From eca7c0f7631cf41b040a1e37fb398669a2f935f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=BD=D0=B5=D0=BA?= <88394552+Lunatuk@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:53:34 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=BF=D0=B5=D1=80=D0=B5=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=204=20=D0=BB=D0=B0=D0=B1=D1=8B=20=D0=B8=20=D1=80=D0=B5=D0=B0?= =?UTF-8?q?=D0=BB=D0=B8=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 31 +- .env.example | 9 +- .github/pull_request_template.md | 68 +- .github/workflows/application.yml | 24 + Dockerfile | 28 + REPORT.md | 189 +- compose.yaml | 219 ++ docs/schema.md | 40 + evidence/docker-checks.txt | 66 + evidence/lab3-checks.json | 1209 +++++++++++ evidence/lab3-checks.txt | 403 ++++ evidence/lab4-checks.json | 2269 ++++++++++++++++++++ evidence/lab4-checks.txt | 533 +++++ evidence/lab5-checks.json | 502 +++++ evidence/lab5-checks.txt | 77 + evidence/lab5-restart.json | 140 ++ evidence/lab5-restart.txt | 4 + evidence/postgres-pause.json | 76 + evidence/query-plans.json | 2621 ++++++++++++++++++++++++ evidence/research.json | 1114 ++++++++++ evidence/research.txt | 32 + infra/haproxy.cfg | 40 + migrations/001_initial.sql | 30 + migrations/002_message_idempotency.sql | 10 + migrations/003_attachments.sql | 34 + migrations/004_attachment_jobs.sql | 42 + requirements.txt | 36 + research/README.md | 12 + research/__init__.py | 0 research/benchmark.py | 168 ++ research/run.py | 99 + scripts/build.sh | 6 + scripts/check-docker.sh | 11 + scripts/check.sh | 17 + scripts/demo.sh | 28 + scripts/init_local.py | 62 + scripts/migrate.sh | 6 + scripts/replicas.py | 90 + scripts/research.sh | 7 + scripts/restart.sh | 6 + scripts/rolling-update.sh | 7 + scripts/run-local.sh | 5 + scripts/start-worker.sh | 5 + scripts/start.sh | 7 + scripts/stop-one.sh | 5 + scripts/stop-worker.sh | 5 + scripts/stop.sh | 6 + scripts/wait-ready.sh | 5 + src/__init__.py | 0 src/__main__.py | 16 + src/app.py | 187 ++ src/attachments.py | 151 ++ src/auth.py | 95 + src/bootstrap_s3.py | 38 + src/broker.py | 38 + src/cleanup.py | 87 + src/config.py | 20 + src/db.py | 52 + src/errors.py | 27 + src/faults.py | 31 + src/healthcheck.py | 15 + src/image_pipeline.py | 70 + src/job_admin.py | 29 + src/jobs.py | 95 + src/migrate.py | 48 + src/models.py | 56 + src/object_store.py | 57 + src/pagination.py | 39 + src/passwords.py | 22 + src/publisher.py | 39 + src/storage.py | 156 ++ src/worker.py | 191 ++ student-tests/attachment_checks.py | 275 +++ student-tests/docker_lifecycle.py | 319 +++ student-tests/helpers.py | 42 + student-tests/lab5_pipeline.py | 324 +++ student-tests/postgres_pause.py | 145 ++ student-tests/test_api.py | 164 ++ student-tests/test_attachments.py | 135 ++ student-tests/test_concurrency.py | 47 + student-tests/test_idempotency.py | 62 + student-tests/test_sessions.py | 51 + student-tests/verify_restart.py | 36 + 83 files changed, 13505 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 docs/schema.md create mode 100644 evidence/docker-checks.txt create mode 100644 evidence/lab3-checks.json create mode 100644 evidence/lab3-checks.txt create mode 100644 evidence/lab4-checks.json create mode 100644 evidence/lab4-checks.txt create mode 100644 evidence/lab5-checks.json create mode 100644 evidence/lab5-checks.txt create mode 100644 evidence/lab5-restart.json create mode 100644 evidence/lab5-restart.txt create mode 100644 evidence/postgres-pause.json create mode 100644 evidence/query-plans.json create mode 100644 evidence/research.json create mode 100644 evidence/research.txt create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_message_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_attachment_jobs.sql create mode 100644 requirements.txt create mode 100644 research/README.md create mode 100644 research/__init__.py create mode 100644 research/benchmark.py create mode 100644 research/run.py create mode 100755 scripts/build.sh create mode 100755 scripts/check-docker.sh create mode 100755 scripts/check.sh create mode 100755 scripts/demo.sh create mode 100644 scripts/init_local.py create mode 100755 scripts/migrate.sh create mode 100644 scripts/replicas.py create mode 100755 scripts/research.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/run-local.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/start.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/stop-worker.sh create mode 100755 scripts/stop.sh create mode 100755 scripts/wait-ready.sh create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/app.py create mode 100644 src/attachments.py create mode 100644 src/auth.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/broker.py create mode 100644 src/cleanup.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/faults.py create mode 100644 src/healthcheck.py create mode 100644 src/image_pipeline.py create mode 100644 src/job_admin.py create mode 100644 src/jobs.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/object_store.py create mode 100644 src/pagination.py create mode 100644 src/passwords.py create mode 100644 src/publisher.py create mode 100644 src/storage.py create mode 100644 src/worker.py create mode 100644 student-tests/attachment_checks.py create mode 100644 student-tests/docker_lifecycle.py create mode 100644 student-tests/helpers.py create mode 100644 student-tests/lab5_pipeline.py create mode 100644 student-tests/postgres_pause.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_sessions.py create mode 100644 student-tests/verify_restart.py diff --git a/.dockerignore b/.dockerignore index bc16e556..40d419ed 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,11 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +* +!Dockerfile +!requirements.txt +!src/ +!src/** +!migrations/ +!migrations/** +!research/ +!research/** +**/__pycache__/ +**/*.py[cod] diff --git a/.env.example b/.env.example index 119384b5..81936309 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,12 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +HOST_PORT=8080 SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Shared sessions carried over from lab4 use Valkey. +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -18,8 +19,12 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_CLEANUP_GRACE_SECONDS=3600 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +JOB_LEASE_SECONDS=90 +JOB_BACKOFF_SECONDS=2 +WORKER_TASK_MEMORY_MB=640 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..1a9eb6b3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,58 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Савельев Иван / БВТ2304 +Целевая оценка (3/4/5): **4** — выполнены требования уровней 3 и 4. +Трек 5, если заявлен: не заявлен. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +### Что реализовано и что перенесено -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Из lab4 перенесены API мессенджера, авторизация и сессии, ACL, идемпотентность сообщений, миграции, приватный S3, две API-реплики за HAProxy, Docker-конфигурация и собственные тесты. + +- Загрузка возвращает `202 queued` после устойчивого сохранения оригинала в S3 staging и транзакции metadata + job + outbox. Статус доступен через GET; content до готовности возвращает 409. +- Добавлены отдельные Valkey Streams broker с AOF и volume, outbox publisher и worker. Доставка at least once, ACK после устойчивого результата; дубли не создают второй набор объектов. +- Worker сохраняет неизменённый оригинал, JPEG thumbnail, центрированный crop 128×128 и optimized WebP по контракту, с учётом EXIF orientation и прозрачности. +- Реализованы пять автоматических попыток, backoff с jitter, lease/recovery, безопасные коды ошибок, реестр failed и явный replay с audit. Ready-задания повторно не запускаются. +- Добавлены ограничения bytes/pixels/времени/памяти, ограниченная конкуренция и очистка staging после фиксации результата либо истечения retention failed. + +### Запуск из чистого clone + +Нужны Docker Engine с Compose v2, Python 3.11, Bash и make. Команды из корня репозитория: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +./scripts/start.sh +./scripts/check.sh + +# Полная проверка на отдельном временном Docker Compose project: +./scripts/check-docker.sh + +# Остановка обычного стенда с сохранением volumes: +./scripts/stop.sh +``` + +`start.sh` создаёт локальные `.env` и secrets, собирает приложение, выполняет миграции и bootstrap, ждёт готовности обеих API-реплик. API доступен на `http://localhost:8080`. + +### Результаты проверок + +Проверено 12.11.2026: `make check validate test`, 39 собственных API-тестов и все 17 групп полного сценария прошли. Воспроизведены остановка worker и рестарты API/broker, оба разрыва outbox, crash worker после PUT и перед ACK, дубли, исчерпание retry/replay, постоянная ошибка, timeout/MemoryError, очистка staging, недоступность broker, persistence и failover API. Отдельно подтверждён фактический restart всех 11 постоянных контейнеров с сохранением их ID и volumes. + +- [Отчёт, архитектура, команды и ограничения](../REPORT.md). +- [Полный вывод проверок](../evidence/lab5-checks.txt) и [JSON результатов](../evidence/lab5-checks.json). +- [Доказательство рестартов](../evidence/lab5-restart.json) и [вывод persistence](../evidence/lab5-restart.txt). + +### Ограничения / известные ошибки + +Стенд учебный, на одном Docker-host; потеря host/disk/volumes и ручное удаление объектов не покрываются. Поддерживается pipeline_version=1; исследовательские треки на 5 не реализованы. При длительном простое worker outbox и staging требуют свободного диска; общей квоты принятой работы нет. Failed staging хранится 24 часа, после удаления replay запрещён. Превышение ресурсов обработки заканчивается failed после ограниченного числа попыток. Старые ready-вложения lab4 остаются доступны без автоматического создания вариантов. + +### Источники и использование ИИ + +Основа — ТЗ, OpenAPI, `attachment-states.md` и `jobs.schema.json` этого репозитория; официальная документация Redis Streams/XAUTOCLAIM, Valkey persistence, Pillow ImageOps и Python resource. Ссылки приведены в [REPORT.md](../REPORT.md#первичные-источники). + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..48e7fc3f --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read + +jobs: + docker: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m venv .venv + - run: .venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt + - run: ./scripts/check-docker.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab5-evidence + path: | + evidence/lab5-checks.txt + evidence/lab5-checks.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..64af5489 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt \ + && python -m pip check + +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid app --no-create-home app + +COPY src/ ./src/ +COPY migrations/ ./migrations/ +COPY research/ ./research/ + +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=2s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-m", "src.healthcheck"] + +CMD ["python", "-m", "src"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..54334b3e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,184 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Савельев Иван, БВТ2304. +- Целевая оценка: **4** — все требования уровней 3 и 4. Исследовательский трек оценки 5 не заявлен. +- Стенд: Docker Compose; HAProxy, две реплики FastAPI, PostgreSQL, Valkey для сессий, SeaweedFS S3; отдельные Valkey Streams broker, outbox publisher и worker. +- Commit/PR: после публикации. Evidence содержит source_commit и working_tree=true: проверялось рабочее дерево поверх указанного commit. +- Использование ИИ: перенос базы, код, проверки и отчёт подготовлены с помощью Codex. Результаты проверок получены реальным запуском Docker, а не предполагаются по коду. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2, Python 3.11, Bash и make. Все команды выполняются из корня репозитория. Внешние заранее установленные БД/брокер/S3 не нужны. -## Архитектура и решения +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +./scripts/start.sh +./scripts/check.sh +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`start.sh` создаёт локальные `.env` и `secrets/`, собирает образ, выполняет миграции и bootstrap приватного bucket, поднимает сервисы и ждёт обе API-реплики через LB. Существующие настройки и пароли сохраняются. `.env`, secrets, виртуальное окружение и runtime data исключены из Git; Docker context содержит только исходники, зависимости и миграции. + +API: http://localhost:8080. Если порт занят, задайте `HOST_PORT=8081` в `.env`; HTTP-проверки запускайте с `BASE_URL=http://localhost:8081`. Compose project `mtusi-lab5`, образы `mtusi-messenger:lab5` и volumes отделены от предыдущих лабораторных. Локальные credentials генерируются заново, файлы из lab4 не копируются. + +```sh +# Полный автоматический сценарий на отдельном временном project и свободном порту: +./scripts/check-docker.sh + +# Обычный стенд, остановка без потери данных: +./scripts/stop.sh +./scripts/start.sh + +# Перезапуск всех постоянных процессов и хранилищ с сохранением volumes: +./scripts/restart.sh + +# Только для намеренного удаления учебных данных ТЕКУЩЕГО Compose project: +docker compose down --volumes +``` + +Полный сценарий создаёт синтетические данные в отдельном `lab5-check-` и удаляет только его контейнеры/volumes. Пишет `evidence/lab5-checks.txt` и `evidence/lab5-checks.json`. Для диагностики сбоя можно явно задать `KEEP_TEST_STACK=1`; это оставит только неуспешный временный стенд. Основной стенд не затрагивается. + +## Что перенесено и изменено + +Из lab4 сохранены регистрация/Argon2id, Basic-вход и Bearer-сессии в Valkey, ACL чатов и вложений, сообщения и keyset-пагинация, X-Request-Id/X-Instance-Id, общая идемпотентность сообщений, две API-реплики за LB, приватный S3, лимиты загрузки, миграции и эксплуатационные скрипты. Предыдущие отчёты — `REPORT.lab2.md`, `REPORT.lab3.md`, `REPORT.lab4.md`; старые evidence относятся к соответствующим работам. + +Объявленный переход API: upload возвращает **202 со снимком queued**, а клиент опрашивает GET metadata. Queued/processing уже можно прикрепить к сообщению в том же чате. Для failed новое прикрепление даёт 409. Content любого не-ready вложения даёт 409, отсутствующий вариант ready — 404. Оригинал скачивается побайтно без перекодирования и redirect; API не раскрывает storage keys/credentials. Контракт, README задания и публичные tests не изменялись. + +Миграции 001–003 сохранены. Миграция 004 добавляет состояния/metadata вариантов и таблицы jobs, outbox, audit событий. Старые ready-вложения сохраняют свои ключи и читаются после обновления; варианты у старых записей не создаются автоматически. Схема описана в `docs/schema.md`. + +## Архитектура и границы устойчивости + +```text +Клиент → HAProxy → API ×2 → PostgreSQL: metadata + job + outbox + → приватный S3 staging + +PostgreSQL outbox → publisher → Valkey Streams (AOF, volume) + ↓ XREADGROUP, prefetch=1 + worker (2 процесса) + ↓ + отдельный resize subprocess + ↓ + S3 original + thumbnail + crop + optimized + ↓ + PostgreSQL COMMIT ready → broker ACK + +cleanup → orphan PUT journal + staging только ready/просроченных failed +``` + +PostgreSQL — источник истины для намерения обработки и состояния. Valkey Streams обеспечивает at-least-once доставку. Это отдельный service `broker`, не Pub/Sub и не тот же экземпляр, что сессии. Stream `attachments:v1`, группа `image-workers`, начало чтения `0`; topology создаётся идемпотентным XGROUP CREATE MKSTREAM. Сообщения не trim-ятся до ACK. После XACK запись удаляется XDEL в одной broker transaction; используется одна группа. + +1. API принимает все bytes с ограничением размера, проверяет имя, MIME и настоящий PNG/JPEG. Resize/генерация вариантов в API не выполняются. +2. Перед PUT фиксируется журнал `attachment_uploads`. Полный проверенный оригинал записывается в S3 `attachments/staging/`. Это тот же приватный S3, но отдельный staging prefix; на этом этапе конечного original/вариантов ещё нет. +3. Одна транзакция PostgreSQL вставляет queued metadata, job и outbox и удаляет запись журнала. **202 отправляется после PUT и успешного COMMIT.** Локальный TemporaryFile нужен только во время HTTP-приёма, устойчивым staging не является. +4. Отдельный publisher читает due outbox с блокировкой SKIP LOCKED и делает XADD. Broker настроен AOF `appendfsync always`, volume, `noeviction`. Успешный XADD подтверждает приём брокером с этой настройкой persistence; это не consumer ACK. После ответа dispatcher отмечает published_at. +5. Worker захватывает job транзакционно: status=processing, случайный lease_token и deadline. Данные для скачивания берёт из БД, а не из сообщения/filename. Проверяет размер и SHA-256 staging. Записывает original и все три варианта, затем в одной транзакции ставит ready и metadata/checksums. Только после COMMIT подтверждает доставку. + +В сообщении только поля `jobs.schema.json`: schema_version, job_id, type, attachment_id, pipeline_version=1, attempt, created_at. Bytes, URL, credentials, filename и storage key отсутствуют. job_id постоянен при redelivery и managed retry; attempt начинается с 1 и меняется при новой управляемой попытке. Native stream ID — отдельный идентификатор доставки. + +## Идемпотентность, retry и replay + +Ключ эффекта `(attachment_id, pipeline_version, variant)` реализован детерминированными объектами: + +```text +attachments//v1/original +attachments//v1/thumbnail +attachments//v1/crop +attachments//v1/optimized +``` + +Число объектов конечного результата — четыре. Дубль queued/processing не захватывает уже арендованную job. Устаревшая попытка не меняет актуальное состояние. Перед PUT и COMMIT worker блокирует строки job/attachment и сверяет lease_token и deadline. При повторе после незавершённого PUT те же детерминированные объекты могут записаться ещё раз, второй набор не создаётся. Дубль ready проверяет полную metadata и размеры объектов через S3 HEAD, затем ACK, не меняя status/updated_at и не выполняя resize. В этой работе поддерживается только pipeline_version=1; трек versioned pipeline на 5 не заявлен. + +Временные ошибки S3, resource limit и timeout дают queued с новым due outbox. Backoff: `uniform(0.5,1.5) × min(60, JOB_BACKOFF_SECONDS × 2^(attempt-1))`; интервал сохраняется в PostgreSQL. Default base=2 s, jitter присутствует. Автоматический бюджет — `JOB_MAX_ATTEMPTS=5`. При исчерпании — failed/error_code=retry_exhausted; последняя безопасная причина сохраняется в job. Нарушение checksum/размера staging или невалидное изображение — постоянный invalid_image, сразу failed без повторов. Тексты исключений и stack traces не передаются в API/registry. + +Publisher также восстанавливает истёкший processing lease: записывает worker_lost и создаёт следующую попытку либо failed при исчерпании бюджета. Это не бесконечный retry. Pending-доставки умершего consumer забираются XAUTOCLAIM после lease timeout. Если recovery уже создал новую попытку, старое сообщение становится obsolete и получает ACK; намерение следующей попытки уже устойчиво сохранено. + +Реестр failed и явный replay: + +```sh +docker compose exec -T worker python -m src.job_admin failed +docker compose exec -T worker python -m src.job_admin replay +``` + +Replay разрешён только для failed с ещё сохранённым staging. Он сохраняет job_id/ключ эффекта, записывает operator_replay в audit, увеличивает глобальный attempt и выделяет новый ограниченный бюджет из пяти попыток. Ready/queued/processing повторять этой командой нельзя; повтор ready завершается ошибкой без изменения БД/объектов. Глобальный attempt монотонен, поэтому старые сообщения не запускают replay как новую работу. После удаления failed staging replay запрещён: исходник надо загрузить заново. HTTP endpoint управления заданиями не добавлен. + +## Правила изображений и ограничения ресурсов + +- Оригинал остаётся исходными bytes, MIME, size_bytes и SHA-256 относятся к нему. +- EXIF orientation применяется к производным изображениям. Прозрачность компонуется на белый; производные RGB, без EXIF/ICC/comment metadata. +- Thumbnail: JPEG quality=85, вписывание в 256×256 без увеличения, целочисленное округление вниз с минимумом 1 px. Fixture 320×200 → 256×160. +- Crop: JPEG quality=85, центрированный квадрат 128×128, LANCZOS; маленький исходник может увеличиваться. +- Optimized: WebP quality=80, method=4; вписывание в 1280×1280 без увеличения. Уменьшение байтов каждого маленького файла не обещается. +- Загрузка ≤10 MiB и ≤40 млн pixels. Limit проверяется до полной распаковки; Content-Length и фактический поток проверяются независимо. Corrupt/MIME spoof — 415, bytes/pixels overflow — 413. API допускает одну проверяемую загрузку на реплику, таймаут тела 30 s и ожидания слота 20 s. +- Worker concurrency=2, каждый consumer получает максимум одну доставку за раз. Для заданного Compose memory budget разрешено concurrency 1 или 2. +- Каждый resize выполняется отдельным subprocess: wall timeout 60 s, CPU limit 60/61 s, address-space limit 640 MiB (`RLIMIT_AS`). При timeout subprocess убивается; MemoryError/resource termination не убивает основной consumer и проходит bounded retry. Worker container ограничен 1536 MiB, 2 CPU и 96 PID. Lease по умолчанию 90 s и должен превышать resize timeout минимум на 5 s. +- S3 операции имеют connect timeout 1 s/read timeout 3 s, SDK retries отключены. PostgreSQL I/O, pool wait и блокировки ограничены. CPU/resize выполняется вне DB-транзакции; транзакция публикации результата включает только fencing, четыре ограниченных PUT и COMMIT. + +`WORKER_TASK_MEMORY_MB` ограничивает адресное пространство процесса, а Compose mem_limit — весь контейнер. Это разные ограничения; поддержка 40 млн pixels не обещает, что любой формат/ориентация уложится в память: превышение ресурсов заканчивается наблюдаемым failed после ограниченного бюджета, а не бесконечной обработкой. + +## Staging retention и очистка + +`cleanup` запускается раз в 30 s. Журнал orphan PUT старше `UPLOAD_CLEANUP_GRACE_SECONDS=3600` очищается под advisory lock: свежая/активная загрузка не удаляется. Crash после staging, но до metadata/job не дал 202; его журнал позволяет удалить orphan без обхода всего bucket. + +Staging queued/processing не удаляется независимо от возраста. После безопасного COMMIT ready staging удаляется при ближайшем проходе; конечные четыре объекта сохраняются. Failed staging и частичные конечные объекты хранятся `STAGING_RETENTION_SECONDS=86400` после последнего перехода в failed, затем удаляются; metadata, failed registry и audit остаются. Операции DELETE идемпотентны, отметка staging_deleted_at фиксируется после удаления. Cleaner и replay блокируют те же строки, поэтому не удаляют staging текущей replay-работы. + +## Сервисы, настройки и bootstrap + +| Роль / service | Версия | Readiness / persistence / credentials | +| --- | --- | --- | +| API / api ×2 | Python 3.11.15; FastAPI 0.142.4 | non-root UID 10001, health live/ready; 384 MiB/1 CPU на реплику; Docker secrets | +| БД / postgres | PostgreSQL 17.6-bookworm | pg_isready, postgres_data; отдельный migrate с checksum/advisory lock | +| Сессии / valkey | Valkey 8.1.3 | PING, AOF always, valkey_data, внутренний порт | +| Брокер / broker | Valkey 8.1.3 | авторизованный PING, AOF always, broker_data, отдельный пароль через secret | +| Outbox / publisher | тот же собранный Python image | restart unless-stopped; DB+broker secrets; topology bootstrap идемпотентен | +| Worker / worker | тот же Python image; Pillow 13.0.0 | restart supervisor, 2 consumer-процесса, leases; DB+broker+S3 secrets | +| S3 / s3 + s3-storage | SeaweedFS 3.85 с digest | gateway probe + filer HTTP; s3_data; storage только в s3-internal | +| S3 bootstrap / s3-bootstrap | MinIO SDK 7.2.20 | root secrets только для создания приватного bucket; повтор безопасен | +| LB / lb | HAProxy 3.2.6-alpine | readiness backend; единственный порт 127.0.0.1:8080 | +| Cleanup / cleanup | тот же Python image | restart unless-stopped, DB+S3 secrets; без broker credentials | + +Python base image закреплён digest в Dockerfile; S3 image — digest в Compose. Все Python-зависимости закреплены в requirements.txt. Административные интерфейсы S3 и внутренние API/БД/broker ports наружу не опубликованы. S3 application credentials разрешают работу только своего bucket/prefix attachments/*; root используется только bootstrap. API/cleaner не получают broker password. `.env.example` содержит пути secret-файлов, а не их значения. + +Соответствие настроек: APP_HOST/APP_PORT — адрес API внутри контейнера; HOST_PORT — порт LB; APP_ENV — учебная метка. SESSION_STORE=valkey и SESSION_TTL_SECONDS=3600 — общее session storage; CACHE_URL_FILE/DATABASE_URL_FILE — Docker secrets. API_REPLICAS=2; IDEMPOTENCY_TTL_SECONDS≥86400. S3_ENDPOINT/REGION/BUCKET и S3_*_FILE — приватный bucket. MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и UPLOAD_CLEANUP_GRACE_SECONDS — приём и orphan cleanup. BROKER_URL_FILE — авторизованный Redis-compatible URL отдельного broker. WORKER_CONCURRENCY, WORKER_TASK_MEMORY_MB, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS, JOB_MAX_ATTEMPTS, JOB_BACKOFF_SECONDS — ограничения/повторы. STAGING_RETENTION_SECONDS — срок хранения failed исходников. + +/health/ready API зависит от PostgreSQL, сессий и S3: они нужны синхронному пути. Broker/worker/publisher не являются зависимостями readiness API: при их остановке original+outbox надёжно принимаются, текстовые сообщения работают, resize синхронно не запускается. У фоновых процессов нет HTTP health endpoint; состояние jobs/pending/failed наблюдается через БД, broker и операторскую CLI. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Проверено 12.11.2026: все **17 групп проверок** и **39 API regression tests** прошли. Docker 29.0.1, aarch64, 10 CPU, 8 217 448 448 bytes RAM Docker VM. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Полный воспроизводимый запуск: `./scripts/check-docker.sh`. Исходник сценария — `student-tests/lab5_pipeline.py`. Результаты и реальные job/attachment IDs — `evidence/lab5-checks.json`, консольный вывод — `evidence/lab5-checks.txt`. Дополнительное доказательство реального restart обычного стенда с прежними container IDs и volumes — `evidence/lab5-restart.json`; воспроизводится `.venv/bin/python student-tests/verify_restart.py`. -## Эксперименты и отказы +| Критерий README | Команда / ключ проверки evidence | Наблюдаемый результат | +| --- | --- | --- | +| Сохранён lab4, переход 201→202, smoke | make check validate test; public_smoke_and_schema, student_api_regressions | Публичный lab5 сценарий и собственные API regression tests | +| Отдельные broker/worker, persistence, лимиты | topology_and_limits | Реальные контейнеры/volumes, две API, broker AOF always, worker concurrency=2/prefetch=1 | +| Worker остановлен → 202 → сообщение → рестарты → ready | worker_offline_api_broker_restart | Тот же attachment/job после рестарта API и broker; 409 до ready; original/три варианта доступны | +| Original/thumbnail/crop/WebP по контракту | image_semantics | Bytes/checksums/MIME и реальные размеры; small/alpha/EXIF/floor cases | +| Outbox, crash COMMIT→publish | outbox_crash_before_publish | Устойчивое намерение доставляется после автоматического restart publisher | +| Неопределённый XADD, дубли | outbox_crash_after_publish | Повтор publish не создаёт второго результата | +| Crash после side effect до COMMIT/ACK | worker_crash_after_objects, worker_crash_before_ack | Lease/reclaim восстанавливают работу; четыре canonical объекта; ready дубль — no-op | +| Retry/backoff/jitter, failed registry, replay | bounded_retry_failed_registry_replay | 5 автоматических неудач → failed; audit содержит время/попытки; явный replay → ready; ready replay отклонён | +| Постоянная ошибка | permanent_error | invalid_image с первой попытки, безопасный код | +| Ограничение времени и памяти обработки | image_subprocess_timeout, image_subprocess_memory | Реальное убийство зависшего subprocess и MemoryError при выделении 1 GiB под лимитом 640 MiB; следующий retry завершает тот же job | +| Безопасная очистка staging | staging_cleanup | Active staging сохранён; ready/expired failed удалены; replay без исходника запрещён | +| Broker недоступен | broker_offline_outbox | API принимает 202 и текст; outbox доставляет после восстановления | +| Сохранность после всех рестартов | make test-persistence; public_persistence | Сессия, история и original переживают рестарты с volumes | +| Отказ API | make test-failover; public_api_failover | Другая реплика продолжает обслуживать подтверждённые сообщения без дублей | -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Failpoints включаются только явным `ENABLE_FAULT_INJECTION=1` во временном тестовом project; по умолчанию отключены. Файлы управления доступны только через docker exec внутри контейнера, HTTP управления ими нет. `before_publish` завершает publisher после DB COMMIT намерения до XADD; `after_publish` — после XADD до фиксации outbox. `after_objects` завершает consumer после PUT четырёх объектов до COMMIT ready; `before_ack` — после COMMIT ready до XACK. Маркеры срабатывания проверяются тестом, поэтому простой быстрый успех без внедрённого сбоя не засчитывается. Отдельный timeout failpoint усыпляет именно ограниченный resize subprocess, не подменяет результат mock-объектом. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Эксперименты и ограничения -## Ограничения и следующие шаги +Это проверка критериев на 4; исследование максимального throughput, сравнение concurrency/prefetch и versioned pipeline не выполнялись. Полный сценарий использует concurrency=2/prefetch=1, max_attempts=5; для быстрых fault checks timeout=5 s, lease=15 s, base backoff=0.15 s. Production-подобные default настройки остаются 60/90/2 s в .env.example. Дата, Docker/hardware и лимиты конкретного запуска записаны в evidence. Измерения latency/RSS этой проверки не трактуются как нагрузочный benchmark. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Обработка at-least-once, не exactly-once: XADD и PUT могут повториться, идемпотентен конечный эффект и его публикация. Сбой host/disk с потерей volumes, потеря единственной PostgreSQL/S3 или ручное удаление объектов не покрываются. Backup/replication этих хранилищ в lab5 не добавлялись. Очередь ограничена maxmemory/noeviction broker; outbox и staging во время длительного простоя требуют диска, общей квоты принятой работы эта оценка не требует. Повторы самого HTTP upload могут создать отдельные вложения по контракту. + +## Первичные источники + +- [Задание и объявленные переходы](README.md), [состояния](contracts/attachment-states.md), [схема job](contracts/jobs.schema.json). +- [Redis Streams: consumer groups, pending, acknowledgements](https://redis.io/docs/latest/develop/data-types/streams/). +- [XAUTOCLAIM: восстановление pending](https://redis.io/docs/latest/commands/xautoclaim/). +- [Valkey persistence: AOF и fsync](https://valkey.io/topics/persistence/). +- [Pillow ImageOps: EXIF transpose и центрированный fit](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html). +- [Python resource limits](https://docs.python.org/3/library/resource.html). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..40eb5de9 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,219 @@ +name: mtusi-lab5 + +x-application: &application + image: mtusi-messenger:lab5 + env_file: .env + environment: &pipeline-environment + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + JOB_BACKOFF_SECONDS: ${JOB_BACKOFF_SECONDS:-2} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640} + ENABLE_FAULT_INJECTION: ${ENABLE_FAULT_INJECTION:-0} + cpus: 1 + mem_limit: 384m + +x-postgres: &postgres + image: postgres:17.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + cpus: 1 + mem_limit: 512m + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + +services: + postgres: + <<: *postgres + volumes: [postgres_data:/var/lib/postgresql/data] + + valkey: + image: valkey/valkey:8.1.3 + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "", "--maxmemory", "64mb", "--maxmemory-policy", "noeviction"] + volumes: [valkey_data:/data] + cpus: 0.5 + mem_limit: 128m + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + broker: + image: valkey/valkey:8.1.3 + command: ["sh", "-c", 'exec valkey-server --appendonly yes --appendfsync always --save "" --maxmemory 128mb --maxmemory-policy noeviction --requirepass "$$(cat /run/secrets/broker_password)"'] + secrets: [broker_password] + volumes: [broker_data:/data] + cpus: 0.5 + mem_limit: 256m + healthcheck: + test: ["CMD-SHELL", 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping | grep -q PONG'] + interval: 2s + timeout: 3s + retries: 30 + + publisher: + <<: *application + command: ["python", "-m", "src.publisher"] + secrets: [database_url, broker_url] + restart: unless-stopped + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + + worker: + <<: *application + command: ["python", "-m", "src.worker"] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + restart: unless-stopped + mem_limit: 1536m + cpus: 2 + pids_limit: 96 + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + stop_grace_period: 8s + + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + environment: + <<: *pipeline-environment + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + depends_on: + postgres: {condition: service_healthy} + valkey: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + s3: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + stop_grace_period: 8s + + migrate: + <<: *application + command: ["python", "-m", "src.migrate"] + secrets: [database_url] + healthcheck: + disable: true + depends_on: + postgres: {condition: service_healthy} + + s3: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + command: ["s3", "-filer=s3-storage:8888", "-ip.bind=0.0.0.0", "-port=9000", "-config=/run/secrets/s3_config"] + secrets: [s3_config] + networks: [default, s3-internal] + depends_on: + s3-storage: {condition: service_healthy} + cpus: 0.5 + mem_limit: 256m + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "9000"] + interval: 2s + timeout: 2s + retries: 30 + + s3-storage: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + command: ["server", "-dir=/data", "-ip=s3-storage", "-ip.bind=0.0.0.0", "-filer", "-master.volumeSizeLimitMB=64", "-volume.max=64", "-volume.preStopSeconds=0"] + volumes: [s3_data:/data] + networks: [s3-internal] + cpus: 1 + mem_limit: 512m + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8888/"] + interval: 2s + timeout: 2s + retries: 30 + + s3-bootstrap: + <<: *application + command: ["python", "-m", "src.bootstrap_s3"] + environment: + S3_ROOT_USER_FILE: /run/secrets/s3_root_user + S3_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + cpus: 0.5 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + s3: {condition: service_healthy} + + cleanup: + <<: *application + command: ["python", "-m", "src.cleanup"] + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + restart: unless-stopped + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + + lb: + image: haproxy:3.2.6-alpine + ports: ["127.0.0.1:${HOST_PORT:-8080}:8080"] + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + cpus: 0.5 + mem_limit: 64m + depends_on: + api: {condition: service_healthy} + stop_grace_period: 10s + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 3 + + postgres-restore: + <<: *postgres + profiles: [research] + volumes: [restore_data:/var/lib/postgresql/data] + +volumes: + postgres_data: + valkey_data: + restore_data: + s3_data: + broker_data: + +networks: + s3-internal: + internal: true + +secrets: + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_config: + file: ./secrets/s3_config diff --git a/docs/schema.md b/docs/schema.md new file mode 100644 index 00000000..2e3cd364 --- /dev/null +++ b/docs/schema.md @@ -0,0 +1,40 @@ +# Схема и транзакции + +Лабораторная 5, оценка 4: клиент → HAProxy → FastAPI × 2 → PostgreSQL + Valkey sessions + приватный S3; outbox publisher → Valkey Streams broker → worker. + +| Таблица PostgreSQL | Назначение и ограничения | +| --- | --- | +| users | UUID PK; username UNIQUE; password_hash Argon2id; display_name; created_at | +| chats | UUID PK; title; created_at | +| chat_members | PK (chat_id, user_id); FK на chats и users | +| messages | UUID PK; FK на чат и членство отправителя; text до 4000 code points; created_at | +| message_attachments | FK на messages и attachments; до 10 позиций; UNIQUE(message_id, position); PK(message_id, attachment_id) | +| attachments | UUID PK; FK chat_id и (chat_id, owner_id); UNIQUE object_key; original metadata; status, variants, error_code, created_at/updated_at | +| attachment_jobs | UUID job_id PK; UNIQUE attachment_id/staging_key; version, attempt/budget, lease, retention | +| attachment_outbox | PK (job_id, attempt); available_at, published_at; индекс неопубликованных | +| attachment_job_events | Audit переходов, safe error_code, attempt, время | +| attachment_uploads | UUID PK, UNIQUE object_key, created_at; долговечный журнал незавершённых PUT | +| message_idempotency | PK (user_id, chat_id, key); request_text, request_attachment_ids, message_id FK, expires_at | +| schema_migrations | Имя и SHA-256 применённой миграции; applied_at | + +В PostgreSQL нет bytes оригинала. `object_key=attachments//v1/original` назначает сервер; filename не участвует в ключе. До ответа 202 в таблице attachments есть status=queued, variants=[], error_code=null. Миграция 004 добавляет status, variants, error_code, updated_at; уже существующие ready-вложения lab4 не меняют ключи и остаются доступны. + +`attachment_jobs` хранит один job_id на attachment_id, staging_key, pipeline_version=1, текущую попытку, бюджет, lease_token/lease_until и отметку удаления staging. `attachment_outbox(job_id,attempt)` создаётся в той же транзакции, что metadata/намерение. Dispatcher отмечает published_at после XADD; потеря ответа вызывает безопасный дубль. Отложенный retry получает отдельную строку outbox с available_at. `attachment_job_events` — audit переходов и явных replay без credentials и текста исключений. + +Данные при 202: приватный S3 `attachments/staging/` и COMMIT metadata+job+outbox. Готовность: worker сохраняет неизменённый original и три варианта по детерминированным ключам, затем атомарно фиксирует status=ready и JSON metadata. До ready content отвечает 409. Worker проверяет lease_token под блокировкой строк перед записью результатов. Cleaner не удаляет staging queued/processing; после ready удаляет staging, а failed хранит до STAGING_RETENTION_SECONDS. + +Индексы: UNIQUE users(username); chat_members(user_id, chat_id); messages(chat_id, created_at, id); attachments(chat_id); attachment_uploads(created_at). Первичные/уникальные ключи индексируют поиск ID и object_key. + +Создание чата и его участников атомарно. Неизвестный участник откатывает весь чат. Сообщение проверяет членство и все attachment_ids в том же чате, затем одной транзакцией записывает messages, message_attachments и результат идемпотентности. Составной FK дополнительно запрещает отправителя вне чата. Пустой/пробельный text разрешён только с вложением: это проверяет модель запроса; FK таблицы message_attachments запрещают несуществующие ссылки. Удаления вложений в HTTP API нет. + +Общая advisory transaction lock сериализует область пользователь + POST + путь чата + Idempotency-Key. Сравниваются text и упорядоченный нормализованный список UUID; отсутствие attachment_ids эквивалентно []. Совпадение возвращает исходный 201 и Message, отличие — 409. TTL не меньше 24 часов. Чтение истории использует keyset (created_at,id), без кэша; на следующем запросе видны COMMIT любой реплики. При конкурентных вставках snapshot между страницами нет; запись с более ранними часами может оказаться до уже пройденного cursor. + +Upload сначала полностью принимает ограниченный поток во временный файл и проверяет PNG/JPEG. Затем отдельной транзакцией фиксирует attachment_uploads. В следующей транзакции берётся advisory lock object_key, выполняется S3 PUT, вставляются queued metadata, job и outbox, удаляется журнал. 202 отправляется после COMMIT. S3 и PostgreSQL не образуют распределённую транзакцию. + +Если PUT прошёл, а COMMIT нет, остаётся журнал. Если исход COMMIT неизвестен, немедленного удаления нет: metadata/job могли сохраниться, а удаление журнала атомарно с ними. Cleaner раз в 30 секунд берёт до 100 записей старше grace period (по умолчанию 3600 s, минимум 60 s), пробует ту же advisory lock и повторно проверяет запись. Активная загрузка пропускается; при наличии ready-метаданных удаляется только запись журнала; иначе удаляется S3 object и затем журнал. DELETE объекта идемпотентен. Сбой S3/БД откладывает обработку; следующий проход повторяет её. Это компенсация журналируемых загрузок уровня 4, без полного обхода bucket. + +Миграции 001 и 002 сохранены. Миграция 003 добавляет вложения, связи, журнал и список вложений в ключе повтора; старые сообщения имеют пустой список. DDL выполняет один migrate-service под общей advisory lock; API проверяют версии и checksum. + +Сервис valkey хранит только серверные сессии; отдельный сервис broker хранит Stream. Сессии: session:SHA256(token) → user_id, expires_at, TTL. Включены AOF appendfsync always и постоянный volume. Сырой токен не записывается в evidence. PostgreSQL, Valkey и S3 bucket проверяются /health/ready; /health/live не обращается к зависимостям. + +Основа схемы перенесена из lab4; команды, измерения и ограничения прежней версии — в [REPORT.lab4.md](../REPORT.lab4.md). Исходные отчёты и результаты lab2/lab3 сохранены отдельно. Текущая лабораторная описывается в [REPORT.md](../REPORT.md). diff --git a/evidence/docker-checks.txt b/evidence/docker-checks.txt new file mode 100644 index 00000000..0ab4442a --- /dev/null +++ b/evidence/docker-checks.txt @@ -0,0 +1,66 @@ +UTC: 2026-10-22T11:17:22.562634+00:00 +Host: Darwin arm64 Python 3.11.9 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Commit: 9961362 + working tree +Command: ./scripts/check-docker.sh; project: lab2-check-ae280d00db +PASS: start.sh built and started the stack, readiness reached +api image=sha256:0b3dc5056cced642eea5ade11d235dce9f69f0d9e6d63956ce9479f2deaa10a0 started=2026-10-22T11:17:27.057932632Z memory=268435456 nano_cpus=1000000000 +postgres image=sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 started=2026-10-22T11:17:24.411773881Z memory=536870912 nano_cpus=1000000000 +valkey image=sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 started=2026-10-22T11:17:24.406509756Z memory=134217728 nano_cpus=500000000 +PASS: non-root UID 10001; Python PID 1; healthcheck; named volumes; no API data volume +Проверяем сервер: http://127.0.0.1:61900 +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:61900" +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 23 tests in 5.743s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. +PASS: demo.sh registered users, logged in and read two messages +PASS: Argon2id in PostgreSQL; logs do not contain message text, passwords or session tokens +PASS: failed membership insert rolled back chat; migrations run twice without duplicate data +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:61900" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +Restart api 2026-10-22T11:17:27.057932632Z -> 2026-10-22T11:17:45.460071626Z +Restart postgres 2026-10-22T11:17:24.411773881Z -> 2026-10-22T11:17:44.637353001Z +Restart valkey 2026-10-22T11:17:24.406509756Z -> 2026-10-22T11:17:44.670095293Z +PASS: API, PostgreSQL and Valkey really restarted; previous token still works +PASS: valkey unavailable -> 503 in 0.11s; anonymous -> 401; readiness 503, liveness 200 +API ready +PASS: valkey recovered without a new login +PASS: postgres unavailable -> 503 in 2.01s; anonymous -> 401; readiness 503, liveness 200 +API ready +PASS: postgres recovered without a new login +API ready +PASS: SESSION_TTL_SECONDS=2; valid token expires without logout +PASS: SIGTERM with unfinished request: 5.54s; new requests refused after 0.16s; exit=0; OOM=false +PASS: all lab2 Docker checks completed +PASS: stop.sh removed containers and preserved named volumes +Removed only temporary Compose project and its test volumes: lab2-check-ae280d00db diff --git a/evidence/lab3-checks.json b/evidence/lab3-checks.json new file mode 100644 index 00000000..8c497137 --- /dev/null +++ b/evidence/lab3-checks.json @@ -0,0 +1,1209 @@ +{ + "utc": "2026-10-29T10:13:09.874202+00:00", + "project": "lab3-check-a31f544685", + "base_url": "http://127.0.0.1:58474", + "host": "Darwin arm64", + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "profile": { + "workers": 4, + "pause_seconds": 0.05, + "operation": "POST message then GET history, same-key retry after uncertain POST" + }, + "compose_version": "2.40.3-desktop.1", + "source_commit": "74aa2242515a05192eceb4e3838e3eb4f032f397", + "working_tree": true, + "containers": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.412188555Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.325770597Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:21.014548544Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "a0a498f6df2fa9b5b03b24cb1ef2f3786139a8f744f71a6803e414867c391f94", + "instance": "a0a498f6df2f", + "service": "migrate", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:14.212270221Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:11.598535928Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:11.59306872Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "api_tests": "public smoke + 28 student tests passed", + "persistence": { + "before": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.412188555Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.325770597Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:21.014548544Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:11.598535928Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:11.59306872Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.473955634Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:35.342063384Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:35.37293605Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "passed": true, + "idempotency_replay_after_restart": true + }, + "sigterm": { + "instance": "7f6a80b69e32", + "unfinished_request": true, + "shutdown_seconds": 5.393, + "new_connections_refused_seconds": 0.167, + "exit_code": 0, + "oom_killed": false + }, + "dependency_readiness": [ + { + "dependency": "valkey", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "postgres", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + } + ], + "public_failover": { + "passed": true, + "containers_after": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:48.981612751Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + ] + }, + "failovers": [ + { + "label": "stop 7f6a80b69e32", + "duration_seconds": 16.009, + "successful_requests": 1576, + "successful_writes": 788, + "errors": 4, + "successful_rps": 98.44, + "first_success_after_action_seconds": { + "POST": 0.014, + "GET": 0.007 + }, + "maximum_success_gap_seconds": { + "POST": 1.957, + "GET": 2.004 + }, + "action_start_seconds": 1.003, + "action_end_seconds": 1.666, + "p50_ms": 7.813, + "p95_ms": 16.393, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "7f6a80b69e32", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 121, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 73, + "errors": 4 + }, + { + "second": 4, + "successes": 109, + "errors": 0 + }, + { + "second": 5, + "successes": 113, + "errors": 0 + }, + { + "second": 6, + "successes": 116, + "errors": 0 + }, + { + "second": 7, + "successes": 113, + "errors": 0 + }, + { + "second": 8, + "successes": 118, + "errors": 0 + }, + { + "second": 9, + "successes": 112, + "errors": 0 + }, + { + "second": 10, + "successes": 102, + "errors": 0 + }, + { + "second": 11, + "successes": 102, + "errors": 0 + }, + { + "second": 12, + "successes": 110, + "errors": 0 + }, + { + "second": 13, + "successes": 116, + "errors": 0 + }, + { + "second": 14, + "successes": 112, + "errors": 0 + }, + { + "second": 15, + "successes": 116, + "errors": 0 + }, + { + "second": 16, + "successes": 0, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.329044791999877, + "GET", + "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.363858082999741, + "POST", + "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.367696166999849, + "POST", + "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4346232499997313, + "GET", + "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:14:23.665074878Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + } + }, + { + "label": "stop f973eaec4d52", + "duration_seconds": 16.015, + "successful_requests": 1574, + "successful_writes": 786, + "errors": 4, + "successful_rps": 98.28, + "first_success_after_action_seconds": { + "POST": 0.002, + "GET": 0.008 + }, + "maximum_success_gap_seconds": { + "POST": 2.006, + "GET": 2.006 + }, + "action_start_seconds": 1.005, + "action_end_seconds": 1.713, + "p50_ms": 7.98, + "p95_ms": 17.217, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "7f6a80b69e32", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 71, + "errors": 4 + }, + { + "second": 4, + "successes": 117, + "errors": 0 + }, + { + "second": 5, + "successes": 115, + "errors": 0 + }, + { + "second": 6, + "successes": 109, + "errors": 0 + }, + { + "second": 7, + "successes": 116, + "errors": 0 + }, + { + "second": 8, + "successes": 109, + "errors": 0 + }, + { + "second": 9, + "successes": 118, + "errors": 0 + }, + { + "second": 10, + "successes": 115, + "errors": 0 + }, + { + "second": 11, + "successes": 115, + "errors": 0 + }, + { + "second": 12, + "successes": 109, + "errors": 0 + }, + { + "second": 13, + "successes": 105, + "errors": 0 + }, + { + "second": 14, + "successes": 113, + "errors": 0 + }, + { + "second": 15, + "successes": 104, + "errors": 0 + }, + { + "second": 16, + "successes": 3, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.3455931250000503, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.349944458000209, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3524864160003744, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.41803337500005, + "GET", + "GET /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + } + ], + "rolling_update": { + "label": "sequential API replacement", + "duration_seconds": 23.774, + "successful_requests": 2131, + "successful_writes": 1063, + "errors": 11, + "successful_rps": 89.64, + "first_success_after_action_seconds": { + "POST": 0.018, + "GET": 0.004 + }, + "maximum_success_gap_seconds": { + "POST": 1.059, + "GET": 2.034 + }, + "action_start_seconds": 1.005, + "action_end_seconds": 19.63, + "p50_ms": 7.723, + "p95_ms": 13.565, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "244bf0b910eb", + "7f6a80b69e32", + "ca9c7a52da66", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 121, + "errors": 0 + }, + { + "second": 1, + "successes": 116, + "errors": 0 + }, + { + "second": 2, + "successes": 109, + "errors": 0 + }, + { + "second": 3, + "successes": 104, + "errors": 0 + }, + { + "second": 4, + "successes": 114, + "errors": 0 + }, + { + "second": 5, + "successes": 33, + "errors": 0 + }, + { + "second": 6, + "successes": 60, + "errors": 4 + }, + { + "second": 7, + "successes": 105, + "errors": 0 + }, + { + "second": 8, + "successes": 117, + "errors": 0 + }, + { + "second": 9, + "successes": 119, + "errors": 0 + }, + { + "second": 10, + "successes": 116, + "errors": 0 + }, + { + "second": 11, + "successes": 115, + "errors": 0 + }, + { + "second": 12, + "successes": 28, + "errors": 0 + }, + { + "second": 13, + "successes": 5, + "errors": 4 + }, + { + "second": 14, + "successes": 74, + "errors": 0 + }, + { + "second": 15, + "successes": 120, + "errors": 0 + }, + { + "second": 16, + "successes": 119, + "errors": 0 + }, + { + "second": 17, + "successes": 115, + "errors": 0 + }, + { + "second": 18, + "successes": 114, + "errors": 0 + }, + { + "second": 19, + "successes": 120, + "errors": 0 + }, + { + "second": 20, + "successes": 116, + "errors": 0 + }, + { + "second": 21, + "successes": 88, + "errors": 0 + }, + { + "second": 22, + "successes": 0, + "errors": 0 + }, + { + "second": 23, + "successes": 3, + "errors": 3 + } + ], + "error_samples": [ + [ + 6.263843457999883, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.297545500000069, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.300835125000049, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.369535666000047, + "GET", + "GET /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [200], received 503" + ], + [ + 13.222746333000032, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ] + ], + "action": { + "before": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:14:47.987597625Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:15:11.976704845Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:14:01.922362174Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:56.000628546Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "ca9c7a52da66ca3c5ffad5cc6668a0d6d60876862683967b6d5f8e438eb630e0", + "instance": "ca9c7a52da66", + "service": "api", + "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", + "started_at": "2026-10-29T10:15:24.374360503Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "244bf0b910eb2fa37937f8f0756a1162e9b339df076f09f4cff90b5c357037a6", + "instance": "244bf0b910eb", + "service": "api", + "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", + "started_at": "2026-10-29T10:15:31.325585882Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:14:01.922362174Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:56.000628546Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ] + } + }, + "passed": true +} diff --git a/evidence/lab3-checks.txt b/evidence/lab3-checks.txt new file mode 100644 index 00000000..27ec14ee --- /dev/null +++ b/evidence/lab3-checks.txt @@ -0,0 +1,403 @@ +Command: ./scripts/check-docker.sh; project: lab3-check-a31f544685 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 1.05kB done +#1 DONE 0.0s + +#2 [migrate internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [api internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [migrate internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [migrate 1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [migrate 2/8] WORKDIR /app +#7 CACHED + +#8 [migrate 3/8] COPY requirements.txt . +#8 CACHED + +#9 [migrate 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#9 CACHED + +#10 [migrate 6/8] COPY src/ ./src/ +#10 CACHED + +#11 [migrate 7/8] COPY migrations/ ./migrations/ +#11 CACHED + +#12 [migrate 5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#12 CACHED + +#13 [migrate 8/8] COPY research/ ./research/ +#13 CACHED + +#14 [api] exporting to image +#14 exporting layers done +#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done +#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done +#14 exporting attestation manifest sha256:352319cde788733f2f237a2a78204630c542e7ca906cff14c902046e8c1d2e5b 0.0s done +#14 exporting manifest list sha256:848e0c76f1678f35dd1b483e7c54552466999551ba7e77c0062bf346671e2902 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.1s + +#15 [migrate] exporting to image +#15 exporting layers done +#15 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done +#15 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done +#15 exporting attestation manifest sha256:b1853c07ddac46c4b398b475052f334aa2753c2fe971e0e4fb2517b0e3e13656 0.0s done +#15 exporting manifest list sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb done +#15 naming to docker.io/library/mtusi-messenger:lab3 done +#15 unpacking to docker.io/library/mtusi-messenger:lab3 done +#15 DONE 0.1s + +#16 [api] resolving provenance for metadata file +#16 DONE 0.0s + +#17 [migrate] resolving provenance for metadata file +#17 DONE 0.0s + mtusi-messenger:lab3 Built + mtusi-messenger:lab3 Built + Network lab3-check-a31f544685_default Creating + Network lab3-check-a31f544685_default Created + Volume lab3-check-a31f544685_postgres_data Creating + Volume lab3-check-a31f544685_postgres_data Created + Volume lab3-check-a31f544685_valkey_data Creating + Volume lab3-check-a31f544685_valkey_data Created + Container lab3-check-a31f544685-valkey-1 Creating + Container lab3-check-a31f544685-postgres-1 Creating + Container lab3-check-a31f544685-valkey-1 Created + Container lab3-check-a31f544685-postgres-1 Created + Container lab3-check-a31f544685-migrate-1 Creating + Container lab3-check-a31f544685-migrate-1 Created + Container lab3-check-a31f544685-api-2 Creating + Container lab3-check-a31f544685-api-1 Creating + Container lab3-check-a31f544685-api-1 Created + Container lab3-check-a31f544685-api-2 Created + Container lab3-check-a31f544685-lb-1 Creating + Container lab3-check-a31f544685-lb-1 Created + Container lab3-check-a31f544685-valkey-1 Starting + Container lab3-check-a31f544685-postgres-1 Starting + Container lab3-check-a31f544685-valkey-1 Started + Container lab3-check-a31f544685-postgres-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-2 Starting + Container lab3-check-a31f544685-api-2 Started + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started + Container lab3-check-a31f544685-api-2 Waiting + Container lab3-check-a31f544685-api-1 Waiting + Container lab3-check-a31f544685-api-2 Healthy + Container lab3-check-a31f544685-api-1 Healthy + Container lab3-check-a31f544685-lb-1 Starting + Container lab3-check-a31f544685-lb-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published +Проверяем сервер: http://127.0.0.1:58474 +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:58474" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 28 tests in 6.783s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. + Container lab3-check-a31f544685-postgres-1 Running + Container lab3-check-a31f544685-postgres-1 Running +PASS: migrations only in one-shot service; repeated execution preserves data +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:58474" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: both API, LB, PostgreSQL, Valkey actually restarted; session and confirmed data preserved +PASS: SIGTERM with unfinished request: {"instance": "7f6a80b69e32", "unfinished_request": true, "shutdown_seconds": 5.393, "new_connections_refused_seconds": 0.167, "exit_code": 0, "oom_killed": false} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 + Container lab3-check-a31f544685-valkey-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopped +PASS: valkey outage: API live=200, ready=503, LB ready=503 + Container lab3-check-a31f544685-valkey-1 Starting + Container lab3-check-a31f544685-valkey-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 + Container lab3-check-a31f544685-postgres-1 Stopping + Container lab3-check-a31f544685-postgres-1 Stopped +PASS: postgres outage: API live=200, ready=503, LB ready=503 + Container lab3-check-a31f544685-postgres-1 Starting + Container lab3-check-a31f544685-postgres-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:58474" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 1, "first_recovery_seconds": 2.452} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Stopped API: 7f6a80b69e32 container=7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6 +{"label": "stop 7f6a80b69e32", "duration_seconds": 16.009, "successful_requests": 1576, "successful_writes": 788, "errors": 4, "successful_rps": 98.44, "first_success_after_action_seconds": {"POST": 0.014, "GET": 0.007}, "maximum_success_gap_seconds": {"POST": 1.957, "GET": 2.004}, "action_start_seconds": 1.003, "action_end_seconds": 1.666, "p50_ms": 7.813, "p95_ms": 16.393, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Stopped API: f973eaec4d52 container=f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b +{"label": "stop f973eaec4d52", "duration_seconds": 16.015, "successful_requests": 1574, "successful_writes": 786, "errors": 4, "successful_rps": 98.28, "first_success_after_action_seconds": {"POST": 0.002, "GET": 0.008}, "maximum_success_gap_seconds": {"POST": 2.006, "GET": 2.006}, "action_start_seconds": 1.005, "action_end_seconds": 1.713, "p50_ms": 7.98, "p95_ms": 17.217, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-2 Starting + Container lab3-check-a31f544685-api-2 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 1.03kB done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 DONE 0.0s + +#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 CACHED + +#8 [3/8] COPY requirements.txt . +#8 CACHED + +#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [2/8] WORKDIR /app +#11 CACHED + +#12 [7/8] COPY migrations/ ./migrations/ +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done +#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done +#14 exporting attestation manifest sha256:3a88ce0975ddcbab53c3ecb185da08ef114e818e9db952983e997cd47add67bf done +#14 exporting manifest list sha256:433a8a3293202a92f092574052b474e8373c0d58649cbcf1c7ebc1af17933d3c done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab3 Built +Local .env and secrets are ready; values are not printed + Container lab3-check-a31f544685-postgres-1 Running +#1 [internal] load local bake definitions +#1 reading from stdin 576B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [3/8] COPY requirements.txt . +#7 CACHED + +#8 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#8 CACHED + +#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [7/8] COPY migrations/ ./migrations/ +#11 CACHED + +#12 [2/8] WORKDIR /app +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done +#14 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done +#14 exporting attestation manifest sha256:25ff7934ca5161da9b6c345af61b6a10b452580e4d349b35939d32feb66664ea done +#14 exporting manifest list sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.0s + +#15 resolving provenance for metadata file +#15 DONE 0.0s +Migrations: already applied +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Replacing API: 7f6a80b69e32 + Container lab3-check-a31f544685-api-2 Running + Container lab3-check-a31f544685-api-3 Creating + Container lab3-check-a31f544685-api-3 Created + Container lab3-check-a31f544685-api-3 Starting + Container lab3-check-a31f544685-api-3 Started +API ready through LB: ca9c7a52da66, f973eaec4d52 +Replacing API: f973eaec4d52 + Container lab3-check-a31f544685-api-3 Running + Container lab3-check-a31f544685-api-4 Creating + Container lab3-check-a31f544685-api-4 Created + Container lab3-check-a31f544685-api-4 Starting + Container lab3-check-a31f544685-api-4 Started +API ready through LB: 244bf0b910eb, ca9c7a52da66 +PASS: all API containers replaced sequentially; stores and LB kept running +{"label": "sequential API replacement", "duration_seconds": 23.774, "successful_requests": 2131, "successful_writes": 1063, "errors": 11, "successful_rps": 89.64, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 1.059, "GET": 2.034}, "action_start_seconds": 1.005, "action_end_seconds": 19.63, "p50_ms": 7.723, "p95_ms": 13.565, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["244bf0b910eb", "7f6a80b69e32", "ca9c7a52da66", "f973eaec4d52"]} +PASS: all lab3 grade-4 checks completed + Container lab3-check-a31f544685-lb-1 Stopping + Container lab3-check-a31f544685-lb-1 Stopped + Container lab3-check-a31f544685-lb-1 Removing + Container lab3-check-a31f544685-lb-1 Removed + Container lab3-check-a31f544685-api-3 Stopping + Container lab3-check-a31f544685-api-4 Stopping + Container lab3-check-a31f544685-api-4 Stopped + Container lab3-check-a31f544685-api-4 Removing + Container lab3-check-a31f544685-api-4 Removed + Container lab3-check-a31f544685-api-3 Stopped + Container lab3-check-a31f544685-api-3 Removing + Container lab3-check-a31f544685-api-3 Removed + Container lab3-check-a31f544685-migrate-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopping + Container lab3-check-a31f544685-migrate-1 Stopped + Container lab3-check-a31f544685-migrate-1 Removing + Container lab3-check-a31f544685-migrate-1 Removed + Container lab3-check-a31f544685-postgres-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopped + Container lab3-check-a31f544685-valkey-1 Removing + Container lab3-check-a31f544685-valkey-1 Removed + Container lab3-check-a31f544685-postgres-1 Stopped + Container lab3-check-a31f544685-postgres-1 Removing + Container lab3-check-a31f544685-postgres-1 Removed + Volume lab3-check-a31f544685_valkey_data Removing + Network lab3-check-a31f544685_default Removing + Volume lab3-check-a31f544685_postgres_data Removing + Volume lab3-check-a31f544685_valkey_data Removed + Volume lab3-check-a31f544685_postgres_data Removed + Network lab3-check-a31f544685_default Removed +Removed only temporary test containers and volumes: lab3-check-a31f544685 diff --git a/evidence/lab4-checks.json b/evidence/lab4-checks.json new file mode 100644 index 00000000..f410993f --- /dev/null +++ b/evidence/lab4-checks.json @@ -0,0 +1,2269 @@ +{ + "utc": "2026-11-06T07:29:29.234494+00:00", + "project": "lab4-check-5aad8e0065", + "base_url": "http://127.0.0.1:57722", + "host": "Darwin arm64", + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "profile": { + "workers": 4, + "pause_seconds": 0.05, + "operation": "POST message then GET history, same-key retry after uncertain POST" + }, + "compose_version": "2.40.3-desktop.1", + "source_commit": "c43d3490a07ad939e4e4748863589b73c84ad05a", + "working_tree": true, + "containers": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.502184793Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.50177996Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:30:03.264509213Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "533e4e511e0441fa577519bbca2a58bc4ceede3ab1660f4c8b026a54b69829f3", + "instance": "533e4e511e04", + "service": "migrate", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:33.838693505Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:29:31.155947879Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:53.783891208Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "813a7d0f6af841b2ed44a684a33baddbbe41405b08d4bdd499ee7ed12b9eccc1", + "instance": "813a7d0f6af8", + "service": "s3-bootstrap", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:56.406424376Z", + "running": false, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_root_user", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_root_password", + "name": null + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:31.154902962Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:29:31.157226212Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "api_tests": "public smoke + student unittest suite passed", + "attachments": { + "private_bucket": { + "anonymous_get": 403, + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "admin_ports_unreachable": [ + 9333, + 8888, + 8080 + ], + "storage_network_isolated": true, + "bootstrap_runs": 2 + }, + "upload_replica_failover": { + "uploaded_by": "ca1f62dfc015", + "stopped_container": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "downloaded_by": "19a474c69210", + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "recovery_seconds": 3.535 + }, + "interrupted_body": { + "metadata_or_journal_created": false + }, + "compensation": { + "failure_after_put": 503, + "orphan_recorded": true, + "fresh_kept": true, + "active_locked_kept": true, + "expired_removed": true, + "ready_kept": true, + "repeat_safe": true, + "automatic_worker": true + }, + "s3_outage": { + "live": 200, + "ready": 503, + "download": 503, + "upload": 503, + "original_preserved_after_recovery": true + }, + "parallel_uploads": { + "concurrency": 10, + "successful": 10, + "errors": 0, + "dimensions": [ + 1600, + 1200 + ], + "size_bytes": 5769335, + "sha256": "b43f2e8293c15183ae478f3bc43cd262e1fa2c532e1c2cfbabceaf3566b4a0d1", + "duration_seconds": 0.914, + "successful_rps": 10.939, + "p50_ms": 610.113, + "p95_ms": 904.09, + "memory_sample_interval_seconds": 0.02, + "memory_includes_monitor_process": true, + "memory": [ + { + "baseline_bytes": 199331840, + "sample_peak_bytes": 215191552, + "lifetime_peak_bytes": 274231296, + "samples": [ + [ + 0.0, + 199098368 + ], + [ + 0.021, + 201449472 + ], + [ + 0.043, + 201371648 + ], + [ + 0.064, + 201367552 + ], + [ + 0.085, + 201625600 + ], + [ + 0.108, + 202936320 + ], + [ + 0.13, + 202309632 + ], + [ + 0.151, + 188710912 + ], + [ + 0.172, + 189378560 + ], + [ + 0.193, + 190976000 + ], + [ + 0.214, + 192249856 + ], + [ + 0.235, + 193933312 + ], + [ + 0.255, + 195235840 + ], + [ + 0.276, + 196837376 + ], + [ + 0.297, + 197255168 + ], + [ + 0.318, + 198422528 + ], + [ + 0.339, + 198475776 + ], + [ + 0.36, + 200818688 + ], + [ + 0.381, + 201170944 + ], + [ + 0.402, + 204890112 + ], + [ + 0.423, + 206053376 + ], + [ + 0.444, + 206901248 + ], + [ + 0.465, + 207024128 + ], + [ + 0.486, + 207085568 + ], + [ + 0.507, + 209682432 + ], + [ + 0.531, + 214925312 + ], + [ + 0.552, + 215191552 + ], + [ + 0.573, + 215191552 + ], + [ + 0.594, + 211496960 + ], + [ + 0.615, + 211415040 + ], + [ + 0.636, + 211611648 + ], + [ + 0.657, + 211410944 + ], + [ + 0.678, + 206876672 + ], + [ + 0.699, + 207892480 + ], + [ + 0.72, + 207699968 + ], + [ + 0.742, + 208068608 + ], + [ + 0.763, + 202301440 + ], + [ + 0.784, + 207642624 + ], + [ + 0.805, + 207413248 + ], + [ + 0.826, + 207593472 + ], + [ + 0.847, + 207593472 + ], + [ + 0.868, + 204062720 + ], + [ + 0.889, + 203718656 + ], + [ + 0.91, + 205983744 + ], + [ + 0.931, + 203411456 + ], + [ + 0.952, + 203411456 + ], + [ + 0.973, + 200871936 + ], + [ + 0.996, + 197525504 + ], + [ + 1.019, + 197496832 + ], + [ + 1.04, + 197496832 + ] + ], + "instance": "19a474c69210", + "limit_bytes": 402653184 + }, + { + "baseline_bytes": 67555328, + "sample_peak_bytes": 103378944, + "lifetime_peak_bytes": 105148416, + "samples": [ + [ + 0.0, + 67555328 + ], + [ + 0.02, + 68780032 + ], + [ + 0.041, + 71090176 + ], + [ + 0.062, + 71782400 + ], + [ + 0.083, + 73428992 + ], + [ + 0.104, + 74698752 + ], + [ + 0.125, + 76619776 + ], + [ + 0.146, + 77070336 + ], + [ + 0.167, + 78118912 + ], + [ + 0.189, + 79024128 + ], + [ + 0.209, + 80576512 + ], + [ + 0.23, + 81674240 + ], + [ + 0.251, + 84914176 + ], + [ + 0.272, + 87511040 + ], + [ + 0.293, + 88616960 + ], + [ + 0.315, + 89174016 + ], + [ + 0.335, + 89919488 + ], + [ + 0.356, + 89976832 + ], + [ + 0.381, + 96006144 + ], + [ + 0.402, + 96436224 + ], + [ + 0.422, + 95522816 + ], + [ + 0.444, + 93450240 + ], + [ + 0.465, + 98467840 + ], + [ + 0.485, + 103378944 + ], + [ + 0.507, + 102744064 + ], + [ + 0.527, + 102805504 + ], + [ + 0.549, + 100847616 + ], + [ + 0.571, + 102080512 + ], + [ + 0.592, + 102064128 + ], + [ + 0.614, + 102064128 + ], + [ + 0.635, + 102318080 + ], + [ + 0.655, + 103100416 + ], + [ + 0.676, + 100167680 + ], + [ + 0.722, + 100024320 + ], + [ + 0.743, + 100548608 + ], + [ + 0.764, + 101707776 + ], + [ + 0.785, + 87347200 + ], + [ + 0.822, + 81424384 + ], + [ + 0.843, + 83718144 + ], + [ + 0.866, + 83685376 + ], + [ + 0.887, + 83664896 + ], + [ + 0.909, + 83668992 + ], + [ + 0.93, + 78086144 + ] + ], + "instance": "ca1f62dfc015", + "limit_bytes": 402653184 + } + ], + "all_original_checksums_verified": true, + "oom_killed": false + } + }, + "persistence": { + "before": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:30:34.948927714Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:01.671594045Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:30:03.264509213Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:29:31.155947879Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:30:53.867775792Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:31.154902962Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:29:31.157226212Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:09.786276466Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:31:09.573287299Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:09.608616716Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:09.641720258Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "passed": true, + "idempotency_replay_after_restart": true + }, + "sigterm": { + "instance": "19a474c69210", + "unfinished_request": true, + "shutdown_seconds": 5.523, + "new_connections_refused_seconds": 0.185, + "exit_code": 0, + "oom_killed": false + }, + "dependency_readiness": [ + { + "dependency": "valkey", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "postgres", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "s3", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + } + ], + "postgres_pause": { + "sql_without_server_timeout": { + "elapsed_seconds": 2.0, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.013 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.107 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.012 + }, + { + "status": 503, + "elapsed_seconds": 2.009 + }, + { + "status": 503, + "elapsed_seconds": 2.095 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.099 + }, + { + "status": 503, + "elapsed_seconds": 2.007 + }, + { + "status": 503, + "elapsed_seconds": 2.102 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 2.008 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 14.572, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 1.975, + "api_not_restarted": true, + "existing_session_and_message_preserved": true, + "passed": true + }, + "public_failover": { + "passed": true, + "containers_after": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:50.739701388Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + ] + }, + "failovers": [ + { + "label": "stop 19a474c69210", + "duration_seconds": 16.012, + "successful_requests": 1420, + "successful_writes": 709, + "errors": 4, + "successful_rps": 88.68, + "first_success_after_action_seconds": { + "POST": 0.051, + "GET": 0.004 + }, + "maximum_success_gap_seconds": { + "POST": 2.002, + "GET": 1.995 + }, + "action_start_seconds": 1.006, + "action_end_seconds": 1.77, + "p50_ms": 10.62, + "p95_ms": 24.67, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "ca1f62dfc015" + ], + "per_second": [ + { + "second": 0, + "successes": 111, + "errors": 0 + }, + { + "second": 1, + "successes": 28, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 70, + "errors": 4 + }, + { + "second": 4, + "successes": 101, + "errors": 0 + }, + { + "second": 5, + "successes": 94, + "errors": 0 + }, + { + "second": 6, + "successes": 106, + "errors": 0 + }, + { + "second": 7, + "successes": 100, + "errors": 0 + }, + { + "second": 8, + "successes": 97, + "errors": 0 + }, + { + "second": 9, + "successes": 107, + "errors": 0 + }, + { + "second": 10, + "successes": 93, + "errors": 0 + }, + { + "second": 11, + "successes": 97, + "errors": 0 + }, + { + "second": 12, + "successes": 101, + "errors": 0 + }, + { + "second": 13, + "successes": 107, + "errors": 0 + }, + { + "second": 14, + "successes": 107, + "errors": 0 + }, + { + "second": 15, + "successes": 101, + "errors": 0 + }, + { + "second": 16, + "successes": 0, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.2868632079917006, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.2872469169960823, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.2959944579924922, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3688164169871015, + "GET", + "GET /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:32:49.121677512Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + } + }, + { + "label": "stop ca1f62dfc015", + "duration_seconds": 16.008, + "successful_requests": 1466, + "successful_writes": 732, + "errors": 4, + "successful_rps": 91.58, + "first_success_after_action_seconds": { + "POST": 0.018, + "GET": 0.028 + }, + "maximum_success_gap_seconds": { + "POST": 2.002, + "GET": 1.999 + }, + "action_start_seconds": 1.002, + "action_end_seconds": 1.832, + "p50_ms": 9.397, + "p95_ms": 18.616, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "ca1f62dfc015" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 52, + "errors": 4 + }, + { + "second": 4, + "successes": 108, + "errors": 0 + }, + { + "second": 5, + "successes": 103, + "errors": 0 + }, + { + "second": 6, + "successes": 106, + "errors": 0 + }, + { + "second": 7, + "successes": 111, + "errors": 0 + }, + { + "second": 8, + "successes": 95, + "errors": 0 + }, + { + "second": 9, + "successes": 103, + "errors": 0 + }, + { + "second": 10, + "successes": 103, + "errors": 0 + }, + { + "second": 11, + "successes": 106, + "errors": 0 + }, + { + "second": 12, + "successes": 102, + "errors": 0 + }, + { + "second": 13, + "successes": 109, + "errors": 0 + }, + { + "second": 14, + "successes": 109, + "errors": 0 + }, + { + "second": 15, + "successes": 103, + "errors": 0 + }, + { + "second": 16, + "successes": 1, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.398277083004359, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.405396166999708, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4118699169921456, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4863260830024956, + "GET", + "GET /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + } + ], + "rolling_update": { + "label": "sequential API replacement", + "duration_seconds": 24.869, + "successful_requests": 2254, + "successful_writes": 1128, + "errors": 8, + "successful_rps": 90.63, + "first_success_after_action_seconds": { + "POST": 0.01, + "GET": 0.021 + }, + "maximum_success_gap_seconds": { + "POST": 1.04, + "GET": 1.04 + }, + "action_start_seconds": 1.009, + "action_end_seconds": 21.862, + "p50_ms": 8.907, + "p95_ms": 16.321, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "63549c0d63ba", + "ca1f62dfc015", + "ed4f3c64a9f6" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 105, + "errors": 0 + }, + { + "second": 2, + "successes": 104, + "errors": 0 + }, + { + "second": 3, + "successes": 101, + "errors": 0 + }, + { + "second": 4, + "successes": 112, + "errors": 0 + }, + { + "second": 5, + "successes": 46, + "errors": 0 + }, + { + "second": 6, + "successes": 44, + "errors": 4 + }, + { + "second": 7, + "successes": 12, + "errors": 0 + }, + { + "second": 8, + "successes": 112, + "errors": 0 + }, + { + "second": 9, + "successes": 110, + "errors": 0 + }, + { + "second": 10, + "successes": 105, + "errors": 0 + }, + { + "second": 11, + "successes": 111, + "errors": 0 + }, + { + "second": 12, + "successes": 107, + "errors": 0 + }, + { + "second": 13, + "successes": 109, + "errors": 0 + }, + { + "second": 14, + "successes": 51, + "errors": 0 + }, + { + "second": 15, + "successes": 43, + "errors": 4 + }, + { + "second": 16, + "successes": 12, + "errors": 0 + }, + { + "second": 17, + "successes": 107, + "errors": 0 + }, + { + "second": 18, + "successes": 101, + "errors": 0 + }, + { + "second": 19, + "successes": 112, + "errors": 0 + }, + { + "second": 20, + "successes": 112, + "errors": 0 + }, + { + "second": 21, + "successes": 106, + "errors": 0 + }, + { + "second": 22, + "successes": 112, + "errors": 0 + }, + { + "second": 23, + "successes": 111, + "errors": 0 + }, + { + "second": 24, + "successes": 97, + "errors": 0 + } + ], + "error_samples": [ + [ + 6.444048083998496, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ], + [ + 6.461145833993214, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ], + [ + 6.465980209002737, + "GET", + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" + ], + [ + 6.52059274999192, + "GET", + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" + ], + [ + 15.429505041989614, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ] + ], + "action": { + "before": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:13.735740384Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:38.250478257Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:32:04.849361089Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:32:11.905651425Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:58.693966586Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "ed4f3c64a9f6b1e8c80f4e061c2017aae179b3feb768d662fd087f2cc96c52fd", + "instance": "ed4f3c64a9f6", + "service": "api", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:50.659312471Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "63549c0d63ba2fcee39baf26eb9ffa0da68e38d2d781a81e4a29f2eff6a0afb8", + "instance": "63549c0d63ba", + "service": "api", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:59.668146169Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:32:04.849361089Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:32:11.905651425Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:58.693966586Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ] + } + }, + "passed": true +} diff --git a/evidence/lab4-checks.txt b/evidence/lab4-checks.txt new file mode 100644 index 00000000..991fcd19 --- /dev/null +++ b/evidence/lab4-checks.txt @@ -0,0 +1,533 @@ +Command: ./scripts/check-docker.sh; project: lab4-check-5aad8e0065 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 564B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 1.01kB done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 CACHED + +#8 [7/8] COPY migrations/ ./migrations/ +#8 CACHED + +#9 [2/8] WORKDIR /app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [3/8] COPY requirements.txt . +#11 CACHED + +#12 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:1146b210a20471d58663fb48a3a24ba22986971d0cce8777f912886768f43306 done +#14 exporting manifest list sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d done +#14 naming to docker.io/library/mtusi-messenger:lab4 done +#14 unpacking to docker.io/library/mtusi-messenger:lab4 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab4 Built + Network lab4-check-5aad8e0065_default Creating + Network lab4-check-5aad8e0065_default Created + Network lab4-check-5aad8e0065_s3-internal Creating + Network lab4-check-5aad8e0065_s3-internal Created + Volume lab4-check-5aad8e0065_valkey_data Creating + Volume lab4-check-5aad8e0065_valkey_data Created + Volume lab4-check-5aad8e0065_postgres_data Creating + Volume lab4-check-5aad8e0065_postgres_data Created + Volume lab4-check-5aad8e0065_s3_data Creating + Volume lab4-check-5aad8e0065_s3_data Created + Container lab4-check-5aad8e0065-s3-storage-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Created + Container lab4-check-5aad8e0065-s3-storage-1 Created + Container lab4-check-5aad8e0065-s3-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Created + Container lab4-check-5aad8e0065-migrate-1 Creating + Container lab4-check-5aad8e0065-s3-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Creating + Container lab4-check-5aad8e0065-migrate-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Created + Container lab4-check-5aad8e0065-api-2 Creating + Container lab4-check-5aad8e0065-api-1 Creating + Container lab4-check-5aad8e0065-cleanup-1 Creating + Container lab4-check-5aad8e0065-api-1 Created + Container lab4-check-5aad8e0065-cleanup-1 Created + Container lab4-check-5aad8e0065-api-2 Created + Container lab4-check-5aad8e0065-lb-1 Creating + Container lab4-check-5aad8e0065-lb-1 Created + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Started + Container lab4-check-5aad8e0065-postgres-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started + Container lab4-check-5aad8e0065-api-2 Started + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started + Container lab4-check-5aad8e0065-api-2 Waiting + Container lab4-check-5aad8e0065-api-1 Waiting + Container lab4-check-5aad8e0065-api-2 Healthy + Container lab4-check-5aad8e0065-api-1 Healthy + Container lab4-check-5aad8e0065-lb-1 Starting + Container lab4-check-5aad8e0065-lb-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published +Проверяем сервер: http://127.0.0.1:57722 +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:57722" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 9.593s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. + Container lab4-check-5aad8e0065-postgres-1 Running + Container lab4-check-5aad8e0065-postgres-1 Running +PASS: migrations only in one-shot service; repeated execution preserves data + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy +PASS: private S3, limited credentials, isolated administration, repeated bootstrap +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +API ready through LB: 19a474c69210 + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: original downloaded through another replica after stopping uploader +PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: S3 outage gives API 503; recovery preserves original and session + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started +PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks: [215191552, 103378944] + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:57722" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved +PASS: SIGTERM with unfinished request: {"instance": "19a474c69210", "unfinished_request": true, "shutdown_seconds": 5.523, "new_connections_refused_seconds": 0.185, "exit_code": 0, "oom_killed": false} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped +PASS: valkey outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped +PASS: postgres outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-postgres-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped +PASS: s3 outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Paused + Container lab4-check-5aad8e0065-postgres-1 Unpaused +PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:57722" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.467} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: 19a474c69210 container=19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75 +{"label": "stop 19a474c69210", "duration_seconds": 16.012, "successful_requests": 1420, "successful_writes": 709, "errors": 4, "successful_rps": 88.68, "first_success_after_action_seconds": {"POST": 0.051, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.995}, "action_start_seconds": 1.006, "action_end_seconds": 1.77, "p50_ms": 10.62, "p95_ms": 24.67, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +{"label": "stop ca1f62dfc015", "duration_seconds": 16.008, "successful_requests": 1466, "successful_writes": 732, "errors": 4, "successful_rps": 91.58, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.028}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.999}, "action_start_seconds": 1.002, "action_end_seconds": 1.832, "p50_ms": 9.397, "p95_ms": 18.616, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 564B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 1.01kB done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 DONE 0.0s + +#7 [3/8] COPY requirements.txt . +#7 CACHED + +#8 [6/8] COPY src/ ./src/ +#8 CACHED + +#9 [2/8] WORKDIR /app +#9 CACHED + +#10 [7/8] COPY migrations/ ./migrations/ +#10 CACHED + +#11 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#11 CACHED + +#12 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:f4a5559518ea1ef0e3747d6084c8ad8dbc8b16dc41cb00062303d0f895b9a235 0.0s done +#14 exporting manifest list sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6 done +#14 naming to docker.io/library/mtusi-messenger:lab4 done +#14 unpacking to docker.io/library/mtusi-messenger:lab4 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab4 Built + Container lab4-check-5aad8e0065-postgres-1 Running +Migrations: already applied +API ready through LB: 19a474c69210, ca1f62dfc015 +Replacing API: 19a474c69210 + Container lab4-check-5aad8e0065-api-2 Running + Container lab4-check-5aad8e0065-api-3 Creating + Container lab4-check-5aad8e0065-api-3 Created + Container lab4-check-5aad8e0065-api-3 Starting + Container lab4-check-5aad8e0065-api-3 Started +API ready through LB: ca1f62dfc015, ed4f3c64a9f6 +Replacing API: ca1f62dfc015 + Container lab4-check-5aad8e0065-api-3 Running + Container lab4-check-5aad8e0065-api-4 Creating + Container lab4-check-5aad8e0065-api-4 Created + Container lab4-check-5aad8e0065-api-4 Starting + Container lab4-check-5aad8e0065-api-4 Started +API ready through LB: 63549c0d63ba, ed4f3c64a9f6 +PASS: all API containers replaced sequentially; stores and LB kept running +{"label": "sequential API replacement", "duration_seconds": 24.869, "successful_requests": 2254, "successful_writes": 1128, "errors": 8, "successful_rps": 90.63, "first_success_after_action_seconds": {"POST": 0.01, "GET": 0.021}, "maximum_success_gap_seconds": {"POST": 1.04, "GET": 1.04}, "action_start_seconds": 1.009, "action_end_seconds": 21.862, "p50_ms": 8.907, "p95_ms": 16.321, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "63549c0d63ba", "ca1f62dfc015", "ed4f3c64a9f6"]} +PASS: all lab4 grade-4 checks completed + Container lab4-check-5aad8e0065-lb-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-lb-1 Stopped + Container lab4-check-5aad8e0065-lb-1 Removing + Container lab4-check-5aad8e0065-lb-1 Removed + Container lab4-check-5aad8e0065-api-3 Stopping + Container lab4-check-5aad8e0065-api-4 Stopping + Container lab4-check-5aad8e0065-api-3 Stopped + Container lab4-check-5aad8e0065-api-3 Removing + Container lab4-check-5aad8e0065-api-3 Removed + Container lab4-check-5aad8e0065-api-4 Stopped + Container lab4-check-5aad8e0065-api-4 Removing + Container lab4-check-5aad8e0065-api-4 Removed + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped + Container lab4-check-5aad8e0065-valkey-1 Removing + Container lab4-check-5aad8e0065-valkey-1 Removed + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-cleanup-1 Removing + Container lab4-check-5aad8e0065-cleanup-1 Removed + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopped + Container lab4-check-5aad8e0065-migrate-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopped + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Removed + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped + Container lab4-check-5aad8e0065-postgres-1 Removing + Container lab4-check-5aad8e0065-postgres-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-1 Removing + Container lab4-check-5aad8e0065-s3-1 Removed + Container lab4-check-5aad8e0065-s3-storage-1 Stopping + Container lab4-check-5aad8e0065-s3-storage-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Removing + Container lab4-check-5aad8e0065-s3-storage-1 Removed + Volume lab4-check-5aad8e0065_valkey_data Removing + Network lab4-check-5aad8e0065_s3-internal Removing + Volume lab4-check-5aad8e0065_postgres_data Removing + Volume lab4-check-5aad8e0065_s3_data Removing + Network lab4-check-5aad8e0065_default Removing + Volume lab4-check-5aad8e0065_valkey_data Removed + Volume lab4-check-5aad8e0065_s3_data Removed + Volume lab4-check-5aad8e0065_postgres_data Removed + Network lab4-check-5aad8e0065_s3-internal Removed + Network lab4-check-5aad8e0065_default Removed +Removed only temporary test containers and volumes: lab4-check-5aad8e0065 diff --git a/evidence/lab5-checks.json b/evidence/lab5-checks.json new file mode 100644 index 00000000..1acd060b --- /dev/null +++ b/evidence/lab5-checks.json @@ -0,0 +1,502 @@ +{ + "utc": "2026-11-12T15:23:27.573351+00:00", + "command": "./scripts/check-docker.sh", + "project": "lab5-check-d233766d34", + "host": "Darwin arm64", + "source_commit": "276dbdad8a34da4c5b57fbf1879ca169b581d75b", + "working_tree": true, + "configuration": { + "concurrency": 2, + "prefetch": 1, + "attempts": 5, + "timeout_seconds": 5, + "lease_seconds": 15, + "backoff_base_seconds": 0.15 + }, + "checks": { + "topology_and_limits": { + "passed": true, + "api_replicas": 2, + "broker": "Valkey Streams with AOF fsync always", + "worker_processes": 2 + }, + "public_smoke_and_schema": { + "passed": true + }, + "student_api_regressions": { + "passed": true + }, + "worker_offline_api_broker_restart": { + "passed": true, + "attachment_id": "63406ab0-cc3c-4fa2-a0aa-e5e3353d5ae3", + "job_id": "7f450ff7-cb2d-4189-a668-f1967dbd94e8", + "message_id": "bc37bafb-efe1-4481-b304-c725c6e8368d", + "variants": { + "thumbnail": { + "size": [ + 256, + 160 + ], + "bytes": 5813 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 2462 + }, + "optimized": { + "size": [ + 320, + 200 + ], + "bytes": 2462 + } + } + }, + "outbox_crash_before_publish": { + "passed": true, + "job_id": "a3973e2b-ff73-4bca-bc41-394f2b875d13", + "automatic_recovery": true + }, + "outbox_crash_after_publish": { + "passed": true, + "job_id": "007cb1d1-4b3b-4d12-9b2b-91a4682e8b08", + "automatic_recovery": true + }, + "worker_crash_after_objects": { + "passed": true, + "job_id": "21129707-7dd0-4a72-ac33-5691dae3194e", + "attempt": 2, + "canonical_objects": 4, + "duplicate_noop": true + }, + "worker_crash_before_ack": { + "passed": true, + "job_id": "2eeba68e-3fc7-42e0-adf9-13ac01c6c982", + "attempt": 1, + "canonical_objects": 4, + "duplicate_noop": true + }, + "bounded_retry_failed_registry_replay": { + "passed": true, + "job_id": "bf756bf3-c3ca-45d2-acdd-3e67eb882d4d", + "max_automatic_attempts": 5, + "audit": [ + { + "event": "processing", + "attempt": 1, + "error_code": null, + "at": "2026-11-12T15:25:18.431558+00:00" + }, + { + "event": "retry", + "attempt": 1, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:18.432821+00:00" + }, + { + "event": "processing", + "attempt": 2, + "error_code": null, + "at": "2026-11-12T15:25:18.692895+00:00" + }, + { + "event": "retry", + "attempt": 2, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:18.69618+00:00" + }, + { + "event": "processing", + "attempt": 3, + "error_code": null, + "at": "2026-11-12T15:25:19.206061+00:00" + }, + { + "event": "retry", + "attempt": 3, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:19.208065+00:00" + }, + { + "event": "processing", + "attempt": 4, + "error_code": null, + "at": "2026-11-12T15:25:19.97778+00:00" + }, + { + "event": "retry", + "attempt": 4, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:19.978984+00:00" + }, + { + "event": "processing", + "attempt": 5, + "error_code": null, + "at": "2026-11-12T15:25:21.777757+00:00" + }, + { + "event": "failed", + "attempt": 5, + "error_code": "retry_exhausted", + "at": "2026-11-12T15:25:21.78129+00:00" + }, + { + "event": "operator_replay", + "attempt": 5, + "error_code": null, + "at": "2026-11-12T15:25:22.680791+00:00" + }, + { + "event": "processing", + "attempt": 6, + "error_code": null, + "at": "2026-11-12T15:25:22.802027+00:00" + }, + { + "event": "ready", + "attempt": 6, + "error_code": null, + "at": "2026-11-12T15:25:23.151043+00:00" + } + ], + "ready_replay_rejected": true + }, + "permanent_error": { + "passed": true, + "job_id": "c2056d32-08bf-4c8a-8980-2f8b6c2511c9", + "attempt": 1, + "error_code": "invalid_image" + }, + "image_subprocess_timeout": { + "passed": true, + "job_id": "6e8efff5-d82d-45f4-af12-3b05dcdb0076", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_subprocess_memory": { + "passed": true, + "job_id": "ff1e7e76-b82d-4ba1-8ca9-79f3d78a4dfe", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_semantics": { + "passed": true, + "cases": [ + { + "case": "small_alpha_on_white", + "variants": { + "thumbnail": { + "size": [ + 17, + 9 + ], + "bytes": 635 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 883 + }, + "optimized": { + "size": [ + 17, + 9 + ], + "bytes": 56 + } + } + }, + { + "case": "exif_orientation_6_and_floor", + "variants": { + "thumbnail": { + "size": [ + 160, + 256 + ], + "bytes": 1270 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 886 + }, + "optimized": { + "size": [ + 201, + 321 + ], + "bytes": 210 + } + } + } + ] + }, + "staging_cleanup": { + "passed": true, + "active_preserved": true, + "orphan_removed": true, + "ready_removed": true, + "expired_failed_removed": true, + "expired_replay_rejected": true + }, + "broker_offline_outbox": { + "passed": true, + "attachment_id": "3b3d3600-efcc-4840-ab76-07868501c21d", + "recovered": true + }, + "public_persistence": { + "passed": true + }, + "public_api_failover": { + "passed": true + } + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "containers": [ + { + "id": "9fbe552bf1bd8c2be76b3434cc5662479ca8d328ca698813ac9a785bcafc0cfd", + "service": "api", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + } + ] + }, + { + "id": "ebc82ed28847cbaf6f474088c68005f05d49469010b45ff3767fb0a944ead6e0", + "service": "api", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "26a48bd7b3390a4cd99667d4d99d6342edce8e26accaee3b181a4d842203261a", + "service": "broker", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_broker_data" + }, + { + "destination": "/run/secrets/broker_password", + "name": null + } + ] + }, + { + "id": "232fdc281b939bf905b4123102a6a6eb609a4ffcb730f6d03707beb07283d69c", + "service": "cleanup", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "e722801286bb112062f1da59af65dc21eddf5059ca086022e0d52f85a0f26154", + "service": "lb", + "image": "haproxy:3.2.6-alpine", + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "ba806a4ce014a516ea867a516743de64bef0d449726bb544d265bf31b4055b1b", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "0fd62d960020290ba4089d05fdcc8da2bb0e2faf7528fc6ed3384e35236d4290", + "service": "postgres", + "image": "postgres:17.6-bookworm", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/var/lib/postgresql/data", + "name": "lab5-check-d233766d34_postgres_data" + }, + { + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "c658ec3ef1e14ca2f1ca8f1530e6f13b37b99ce8389fafeae84e841e65daa2fc", + "service": "publisher", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "id": "bc944a3a03374052aba43f42767c253755a921090fd15ead92fa2a924507c941", + "service": "s3", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "destination": "/data", + "name": "0496c4617fd5cac2843b4c391fc1eb42e15c90c4d9a35d092e71a6fd8815960f" + } + ] + }, + { + "id": "9e7382c34331cb65974fa3feb248105c8055fa25c251151f0f004b65fc388923", + "service": "s3-bootstrap", + "image": "mtusi-messenger:lab5", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_root_password", + "name": null + }, + { + "destination": "/run/secrets/s3_root_user", + "name": null + } + ] + }, + { + "id": "b9821d3ec48ce18b8255a446cf99b02d6ea9658de406a15cc0ace7ccf7d7350e", + "service": "s3-storage", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_s3_data" + } + ] + }, + { + "id": "c8ddc9fede2be42d05e79147dfc7dd87efede3b67f3226afeee1161fcaa49428", + "service": "valkey", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_valkey_data" + } + ] + }, + { + "id": "9fee2e7832c5c11e81bb56df9d435ad090fdb1fddb85efbf014ea6019559ed75", + "service": "worker", + "image": "mtusi-messenger:lab5", + "memory_bytes": 1610612736, + "nano_cpus": 2000000000, + "mounts": [ + { + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + } + ], + "passed": true +} diff --git a/evidence/lab5-checks.txt b/evidence/lab5-checks.txt new file mode 100644 index 00000000..61c12416 --- /dev/null +++ b/evidence/lab5-checks.txt @@ -0,0 +1,77 @@ +PASS: topology_and_limits {"api_replicas": 2, "broker": "Valkey Streams with AOF fsync always", "worker_processes": 2} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:57277" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +PASS: public_smoke_and_schema {} +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 6.981s + +OK +PASS: student_api_regressions {} +PASS: worker_offline_api_broker_restart {"attachment_id": "63406ab0-cc3c-4fa2-a0aa-e5e3353d5ae3", "job_id": "7f450ff7-cb2d-4189-a668-f1967dbd94e8", "message_id": "bc37bafb-efe1-4481-b304-c725c6e8368d", "variants": {"thumbnail": {"size": [256, 160], "bytes": 5813}, "crop": {"size": [128, 128], "bytes": 2462}, "optimized": {"size": [320, 200], "bytes": 2462}}} +PASS: outbox_crash_before_publish {"job_id": "a3973e2b-ff73-4bca-bc41-394f2b875d13", "automatic_recovery": true} +PASS: outbox_crash_after_publish {"job_id": "007cb1d1-4b3b-4d12-9b2b-91a4682e8b08", "automatic_recovery": true} +PASS: worker_crash_after_objects {"job_id": "21129707-7dd0-4a72-ac33-5691dae3194e", "attempt": 2, "canonical_objects": 4, "duplicate_noop": true} +PASS: worker_crash_before_ack {"job_id": "2eeba68e-3fc7-42e0-adf9-13ac01c6c982", "attempt": 1, "canonical_objects": 4, "duplicate_noop": true} +PASS: bounded_retry_failed_registry_replay {"job_id": "bf756bf3-c3ca-45d2-acdd-3e67eb882d4d", "max_automatic_attempts": 5, "audit": [{"event": "processing", "attempt": 1, "error_code": null, "at": "2026-11-12T15:25:18.431558+00:00"}, {"event": "retry", "attempt": 1, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:18.432821+00:00"}, {"event": "processing", "attempt": 2, "error_code": null, "at": "2026-11-12T15:25:18.692895+00:00"}, {"event": "retry", "attempt": 2, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:18.69618+00:00"}, {"event": "processing", "attempt": 3, "error_code": null, "at": "2026-11-12T15:25:19.206061+00:00"}, {"event": "retry", "attempt": 3, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:19.208065+00:00"}, {"event": "processing", "attempt": 4, "error_code": null, "at": "2026-11-12T15:25:19.97778+00:00"}, {"event": "retry", "attempt": 4, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:19.978984+00:00"}, {"event": "processing", "attempt": 5, "error_code": null, "at": "2026-11-12T15:25:21.777757+00:00"}, {"event": "failed", "attempt": 5, "error_code": "retry_exhausted", "at": "2026-11-12T15:25:21.78129+00:00"}, {"event": "operator_replay", "attempt": 5, "error_code": null, "at": "2026-11-12T15:25:22.680791+00:00"}, {"event": "processing", "attempt": 6, "error_code": null, "at": "2026-11-12T15:25:22.802027+00:00"}, {"event": "ready", "attempt": 6, "error_code": null, "at": "2026-11-12T15:25:23.151043+00:00"}], "ready_replay_rejected": true} +PASS: permanent_error {"job_id": "c2056d32-08bf-4c8a-8980-2f8b6c2511c9", "attempt": 1, "error_code": "invalid_image"} +PASS: image_subprocess_timeout {"job_id": "6e8efff5-d82d-45f4-af12-3b05dcdb0076", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_subprocess_memory {"job_id": "ff1e7e76-b82d-4ba1-8ca9-79f3d78a4dfe", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_semantics {"cases": [{"case": "small_alpha_on_white", "variants": {"thumbnail": {"size": [17, 9], "bytes": 635}, "crop": {"size": [128, 128], "bytes": 883}, "optimized": {"size": [17, 9], "bytes": 56}}}, {"case": "exif_orientation_6_and_floor", "variants": {"thumbnail": {"size": [160, 256], "bytes": 1270}, "crop": {"size": [128, 128], "bytes": 886}, "optimized": {"size": [201, 321], "bytes": 210}}}]} +PASS: staging_cleanup {"active_preserved": true, "orphan_removed": true, "ready_removed": true, "expired_failed_removed": true, "expired_replay_rejected": true} +PASS: broker_offline_outbox {"attachment_id": "3b3d3600-efcc-4840-ab76-07868501c21d", "recovered": true} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:57277" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_persistence {} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:57277" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.797} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_api_failover {} +PASS: all lab5 grade-4 criteria completed +Removed isolated test stack: lab5-check-d233766d34 diff --git a/evidence/lab5-restart.json b/evidence/lab5-restart.json new file mode 100644 index 00000000..0bca0234 --- /dev/null +++ b/evidence/lab5-restart.json @@ -0,0 +1,140 @@ +{ + "utc": "2026-11-12T15:28:31.543303+00:00", + "command": ".venv/bin/python student-tests/verify_restart.py", + "passed": true, + "before": { + "9e5a9d3d2bf0b2e47ee6d85b40b1899881db0d4d91d9ec09e50fb36aca67833c": { + "service": "api", + "started_at": "2026-11-12T15:28:13.153166256Z", + "volumes": [] + }, + "83bab824b05accd2aa8829e83e272d72d4105644f8270cf3dc15f3412d569f5b": { + "service": "api", + "started_at": "2026-11-12T15:28:13.038450506Z", + "volumes": [] + }, + "6f72735cbaed44f82555dcd550199764ab53a8c61a63c88f25c612fd39b6baba": { + "service": "broker", + "started_at": "2026-11-12T15:27:46.698266716Z", + "volumes": [ + "mtusi-lab5_broker_data" + ] + }, + "0f9060c7dc8473fa926c5affc560b3478b90c77771de9810ebba7858a30838d5": { + "service": "cleanup", + "started_at": "2026-11-12T15:28:13.036348173Z", + "volumes": [] + }, + "d381adfb7273a5dd2f22d361a6e1ff490e5c2cf44ff147a8253f7f940cd01de4": { + "service": "lb", + "started_at": "2026-11-12T15:28:18.76757655Z", + "volumes": [] + }, + "9906c3edca610723e6f09b64867fa71a543f18ef4b474804414e9f818a803115": { + "service": "postgres", + "started_at": "2026-11-12T15:27:46.700095216Z", + "volumes": [ + "mtusi-lab5_postgres_data" + ] + }, + "d705dceffd740900bbad04ee520bb96247f286c7c5381734a28bc1c9e30f27e4": { + "service": "publisher", + "started_at": "2026-11-12T15:27:49.924416218Z", + "volumes": [] + }, + "cbe9063ff82960d9db58bc5713af90e4be3e72fdd1c08b0a92a93849dc6ad1aa": { + "service": "s3", + "started_at": "2026-11-12T15:28:09.808290754Z", + "volumes": [ + "fe34bef7724beefe41335cd197f83aae91b16df70766cee6321bff12f765bc66" + ] + }, + "6d1b6c402e19a068a0f2f4b9c06d5abbec7e41b5d778a883e6d045be409f6d4c": { + "service": "s3-storage", + "started_at": "2026-11-12T15:27:46.699445508Z", + "volumes": [ + "mtusi-lab5_s3_data" + ] + }, + "6d8c3e69c4b60acaf42e0259d367a0a3f2c06081a6349c284ddf9f2e9b5ba399": { + "service": "valkey", + "started_at": "2026-11-12T15:27:46.697619716Z", + "volumes": [ + "mtusi-lab5_valkey_data" + ] + }, + "28fabe35a9a5b57e0a3a96bb4ae5308fbd2fa825470799180a2471b55400206e": { + "service": "worker", + "started_at": "2026-11-12T15:28:13.037172673Z", + "volumes": [] + } + }, + "after": { + "9e5a9d3d2bf0b2e47ee6d85b40b1899881db0d4d91d9ec09e50fb36aca67833c": { + "service": "api", + "started_at": "2026-11-12T15:28:33.373854585Z", + "volumes": [] + }, + "83bab824b05accd2aa8829e83e272d72d4105644f8270cf3dc15f3412d569f5b": { + "service": "api", + "started_at": "2026-11-12T15:28:33.338416001Z", + "volumes": [] + }, + "6f72735cbaed44f82555dcd550199764ab53a8c61a63c88f25c612fd39b6baba": { + "service": "broker", + "started_at": "2026-11-12T15:28:33.257433543Z", + "volumes": [ + "mtusi-lab5_broker_data" + ] + }, + "0f9060c7dc8473fa926c5affc560b3478b90c77771de9810ebba7858a30838d5": { + "service": "cleanup", + "started_at": "2026-11-12T15:28:40.990298547Z", + "volumes": [] + }, + "d381adfb7273a5dd2f22d361a6e1ff490e5c2cf44ff147a8253f7f940cd01de4": { + "service": "lb", + "started_at": "2026-11-12T15:28:33.069501668Z", + "volumes": [] + }, + "9906c3edca610723e6f09b64867fa71a543f18ef4b474804414e9f818a803115": { + "service": "postgres", + "started_at": "2026-11-12T15:28:33.134879876Z", + "volumes": [ + "mtusi-lab5_postgres_data" + ] + }, + "d705dceffd740900bbad04ee520bb96247f286c7c5381734a28bc1c9e30f27e4": { + "service": "publisher", + "started_at": "2026-11-12T15:28:40.90857463Z", + "volumes": [] + }, + "cbe9063ff82960d9db58bc5713af90e4be3e72fdd1c08b0a92a93849dc6ad1aa": { + "service": "s3", + "started_at": "2026-11-12T15:28:33.167460626Z", + "volumes": [ + "fe34bef7724beefe41335cd197f83aae91b16df70766cee6321bff12f765bc66" + ] + }, + "6d1b6c402e19a068a0f2f4b9c06d5abbec7e41b5d778a883e6d045be409f6d4c": { + "service": "s3-storage", + "started_at": "2026-11-12T15:28:40.950084172Z", + "volumes": [ + "mtusi-lab5_s3_data" + ] + }, + "6d8c3e69c4b60acaf42e0259d367a0a3f2c06081a6349c284ddf9f2e9b5ba399": { + "service": "valkey", + "started_at": "2026-11-12T15:28:33.205284001Z", + "volumes": [ + "mtusi-lab5_valkey_data" + ] + }, + "28fabe35a9a5b57e0a3a96bb4ae5308fbd2fa825470799180a2471b55400206e": { + "service": "worker", + "started_at": "2026-11-12T15:28:33.299354043Z", + "volumes": [] + } + }, + "same_containers_and_volumes": true +} diff --git a/evidence/lab5-restart.txt b/evidence/lab5-restart.txt new file mode 100644 index 00000000..4f743ef4 --- /dev/null +++ b/evidence/lab5-restart.txt @@ -0,0 +1,4 @@ +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: all persistent services actually restarted; container IDs and volumes preserved diff --git a/evidence/postgres-pause.json b/evidence/postgres-pause.json new file mode 100644 index 00000000..73382716 --- /dev/null +++ b/evidence/postgres-pause.json @@ -0,0 +1,76 @@ +{ + "utc": "2026-11-06T07:27:40.514878+00:00", + "project": "lab4-pg-pause-30f1d978ce", + "command": ".venv/bin/python student-tests/postgres_pause.py", + "passed": true, + "sql_without_server_timeout": { + "elapsed_seconds": 2.001, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.012 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.11 + }, + { + "status": 503, + "elapsed_seconds": 2.011 + }, + { + "status": 503, + "elapsed_seconds": 2.014 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.194 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.299 + }, + { + "status": 503, + "elapsed_seconds": 2.203 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 0.009 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 12.565, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 2.874, + "api_not_restarted": true, + "existing_session_and_message_preserved": true +} diff --git a/evidence/query-plans.json b/evidence/query-plans.json new file mode 100644 index 00000000..25f478d6 --- /dev/null +++ b/evidence/query-plans.json @@ -0,0 +1,2621 @@ +[ + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5732.59, + "Total Cost": 5738.45, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 7.421, + "Actual Total Time": 8.522, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5732.59, + "Total Cost": 10128.12, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 7.421, + "Actual Total Time": 8.508, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4732.58, + "Total Cost": 4828.13, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 6.621, + "Actual Total Time": 6.624, + "Actual Rows": 51, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 0.215, + "Actual Total Time": 4.164, + "Actual Rows": 31500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.007+00'::timestamp with time zone, '9cc0c1db-51d1-5bf2-e5a4-83cb93303974'::uuid)))", + "Rows Removed by Filter": 18500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.031, + "Triggers": [], + "Execution Time": 8.532 + } + }, + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5354.06, + "Total Cost": 5359.93, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 5.792, + "Actual Total Time": 7.033, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5354.06, + "Total Cost": 8444.8, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 5.792, + "Actual Total Time": 7.028, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4354.05, + "Total Cost": 4421.24, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 4.884, + "Actual Total Time": 4.886, + "Actual Rows": 40, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 1.017, + "Actual Total Time": 3.623, + "Actual Rows": 17500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.035+00'::timestamp with time zone, '169ee7a7-ca0d-aa05-bd8c-a1eb275708e7'::uuid)))", + "Rows Removed by Filter": 32500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.038, + "Triggers": [], + "Execution Time": 7.042 + } + }, + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4967.83, + "Total Cost": 4973.69, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 25.942, + "Actual Total Time": 27.248, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4967.83, + "Total Cost": 6727.21, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 25.942, + "Actual Total Time": 27.243, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3967.82, + "Total Cost": 4006.07, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 14.755, + "Actual Total Time": 14.763, + "Actual Rows": 26, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 25, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 13.398, + "Actual Total Time": 14.487, + "Actual Rows": 3500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.063+00'::timestamp with time zone, '987963b5-9cc6-d246-e263-52373e7f4b96'::uuid)))", + "Rows Removed by Filter": 46500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.047, + "Triggers": [], + "Execution Time": 27.257 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3191.11, + "Total Cost": 3191.24, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 3.535, + "Actual Total Time": 3.542, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3191.11, + "Total Cost": 3205.3, + "Plan Rows": 5677, + "Plan Width": 160, + "Actual Startup Time": 3.534, + "Actual Total Time": 3.538, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.91, + "Total Cost": 3001.71, + "Plan Rows": 5677, + "Plan Width": 160, + "Actual Startup Time": 0.442, + "Actual Total Time": 2.23, + "Actual Rows": 18000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.072+00'::timestamp with time zone, '97b900c6-4a0c-f323-4d98-d97c50488136'::uuid))", + "Rows Removed by Filter": 2000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.29, + "Actual Total Time": 0.291, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.043, + "Triggers": [], + "Execution Time": 3.549 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3136.93, + "Total Cost": 3137.05, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 2.648, + "Actual Total Time": 2.654, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3136.93, + "Total Cost": 3147.09, + "Plan Rows": 4065, + "Plan Width": 160, + "Actual Startup Time": 2.647, + "Actual Total Time": 2.65, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.51, + "Total Cost": 3001.31, + "Plan Rows": 4065, + "Plan Width": 160, + "Actual Startup Time": 0.94, + "Actual Total Time": 1.93, + "Actual Rows": 10000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.08+00'::timestamp with time zone, '9042bcb2-df86-667b-9e54-e6e6c0c7923a'::uuid))", + "Rows Removed by Filter": 10000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.276, + "Actual Total Time": 0.276, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.03, + "Triggers": [], + "Execution Time": 2.66 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3081.5, + "Total Cost": 3081.63, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.813, + "Actual Total Time": 1.819, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3081.5, + "Total Cost": 3087.54, + "Plan Rows": 2416, + "Plan Width": 160, + "Actual Startup Time": 1.813, + "Actual Total Time": 1.816, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.1, + "Total Cost": 3000.9, + "Plan Rows": 2416, + "Plan Width": 160, + "Actual Startup Time": 1.459, + "Actual Total Time": 1.667, + "Actual Rows": 2000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.088+00'::timestamp with time zone, '81028714-cb0a-8e0f-6d74-4c379351efc2'::uuid))", + "Rows Removed by Filter": 18000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.277, + "Actual Total Time": 0.277, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.027, + "Triggers": [], + "Execution Time": 1.827 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2710.71, + "Total Cost": 2710.84, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.58, + "Actual Total Time": 1.586, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2710.71, + "Total Cost": 2712.76, + "Plan Rows": 818, + "Plan Width": 160, + "Actual Startup Time": 1.58, + "Actual Total Time": 1.582, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.37, + "Total Cost": 2683.42, + "Plan Rows": 818, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 1.001, + "Actual Rows": 8100, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0909+00'::timestamp with time zone, 'b4d17acc-f297-d5d5-746e-2ef777eaa364'::uuid))", + "Rows Removed by Filter": 900, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.121, + "Actual Total Time": 0.121, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.021, + "Triggers": [], + "Execution Time": 1.592 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2699.82, + "Total Cost": 2699.95, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.21, + "Actual Total Time": 1.216, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2699.82, + "Total Cost": 2701.06, + "Plan Rows": 494, + "Plan Width": 160, + "Actual Startup Time": 1.209, + "Actual Total Time": 1.212, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.29, + "Total Cost": 2683.34, + "Plan Rows": 494, + "Plan Width": 160, + "Actual Startup Time": 0.437, + "Actual Total Time": 0.889, + "Actual Rows": 4500, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0945+00'::timestamp with time zone, '64cc6094-35b0-52bc-2c33-655d810cbabc'::uuid))", + "Rows Removed by Filter": 4500, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.121, + "Actual Total Time": 0.121, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 1.222 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2689.0, + "Total Cost": 2689.13, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.834, + "Actual Total Time": 0.84, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2689.0, + "Total Cost": 2689.43, + "Plan Rows": 172, + "Plan Width": 160, + "Actual Startup Time": 0.834, + "Actual Total Time": 0.836, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.21, + "Total Cost": 2683.26, + "Plan Rows": 172, + "Plan Width": 160, + "Actual Startup Time": 0.661, + "Actual Total Time": 0.766, + "Actual Rows": 900, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0981+00'::timestamp with time zone, '8de6f8f9-25ec-97e3-e3b7-59eac1d2fd4c'::uuid))", + "Rows Removed by Filter": 8100, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.12, + "Actual Total Time": 0.12, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.035, + "Triggers": [], + "Execution Time": 0.846 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.42, + "Total Cost": 1993.45, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 0.198, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.42, + "Total Cost": 1993.45, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 0.194, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.024, + "Actual Total Time": 0.125, + "Actual Rows": 900, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0991+00'::timestamp with time zone, 'a1580a3c-2826-e768-e4ea-f6c5a0e7171b'::uuid))", + "Rows Removed by Filter": 100, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.204 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.31, + "Total Cost": 1993.32, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.139, + "Actual Total Time": 0.145, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.31, + "Total Cost": 1993.32, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.139, + "Actual Total Time": 0.141, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.051, + "Actual Total Time": 0.1, + "Actual Rows": 500, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0995+00'::timestamp with time zone, '86263544-3ca9-88ac-6a18-01fe3ed3d579'::uuid))", + "Rows Removed by Filter": 500, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.019, + "Triggers": [], + "Execution Time": 0.151 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.24, + "Total Cost": 1993.24, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.095, + "Actual Total Time": 0.101, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.24, + "Total Cost": 1993.24, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.095, + "Actual Total Time": 0.097, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "quicksort", + "Sort Space Used": 50, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.077, + "Actual Total Time": 0.086, + "Actual Rows": 100, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0999+00'::timestamp with time zone, 'dc2ec078-4b31-5bfc-d9ac-d4e6671f761b'::uuid))", + "Rows Removed by Filter": 900, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.106 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 10.63, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 13039.29, + "Plan Rows": 65139, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.007+00'::timestamp with time zone, '9cc0c1db-51d1-5bf2-e5a4-83cb93303974'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.021 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 13.82, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 11922.0, + "Plan Rows": 45372, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.035+00'::timestamp with time zone, '169ee7a7-ca0d-aa05-bd8c-a1eb275708e7'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 21.74, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 10814.04, + "Plan Rows": 25860, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.063+00'::timestamp with time zone, '987963b5-9cc6-d246-e263-52373e7f4b96'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 88.93, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 9665.82, + "Plan Rows": 5569, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.072+00'::timestamp with time zone, '97b900c6-4a0c-f323-4d98-d97c50488136'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 111.12, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 8668.14, + "Plan Rows": 3993, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.08+00'::timestamp with time zone, '9042bcb2-df86-667b-9e54-e6e6c0c7923a'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.016, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 134.87, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 6322.47, + "Plan Rows": 2398, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.088+00'::timestamp with time zone, '81028714-cb0a-8e0f-6d74-4c379351efc2'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 171.67, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 2737.17, + "Plan Rows": 815, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0909+00'::timestamp with time zone, 'b4d17acc-f297-d5d5-746e-2ef777eaa364'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 182.11, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 1767.48, + "Plan Rows": 496, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0945+00'::timestamp with time zone, '64cc6094-35b0-52bc-2c33-655d810cbabc'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.019, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 194.46, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 635.83, + "Plan Rows": 167, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0981+00'::timestamp with time zone, '8de6f8f9-25ec-97e3-e3b7-59eac1d2fd4c'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.021, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 39.4, + "Plan Rows": 9, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 39.4, + "Plan Rows": 9, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0991+00'::timestamp with time zone, 'a1580a3c-2826-e768-e4ea-f6c5a0e7171b'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 23.92, + "Plan Rows": 5, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 23.92, + "Plan Rows": 5, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0995+00'::timestamp with time zone, '86263544-3ca9-88ac-6a18-01fe3ed3d579'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.018, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 8.44, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.012, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 8.44, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.008, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0999+00'::timestamp with time zone, 'dc2ec078-4b31-5bfc-d9ac-d4e6671f761b'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.017 + } + } +] \ No newline at end of file diff --git a/evidence/research.json b/evidence/research.json new file mode 100644 index 00000000..5074a7a0 --- /dev/null +++ b/evidence/research.json @@ -0,0 +1,1114 @@ +{ + "postgres": "PostgreSQL 17.6 (Debian 17.6-2.pgdg12+1) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", + "messages": 100000, + "chat_sizes": [ + 70000, + 20000, + 9000, + 1000 + ], + "repeats": 3, + "warmup_per_case": 1, + "measurements_per_case": 5, + "page_size": 51, + "query": "SELECT * FROM messages WHERE chat_id=%s AND (created_at, id)>(%s, %s)\n ORDER BY created_at, id LIMIT %s", + "summary": { + "chat_only": { + "insert_100000_median_ms": 1386.7587500000127, + "index_bytes": 671744, + "wal_bytes_median": 38822560, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 1.6852079999978287, + "p95_ms": 7.566457999985232, + "p99_ms": 46.40233399999261, + "successful_sql_rps": 282.01364182715855 + }, + "history": { + "insert_100000_median_ms": 1404.4910420000178, + "index_bytes": 7315456, + "wal_bytes_median": 43108512, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 0.22610399997802233, + "p95_ms": 0.29704099995342403, + "p99_ms": 0.3384169999662845, + "successful_sql_rps": 4322.810483886352 + } + }, + "trials": [ + { + "variant": "chat_only", + "repeat": 1, + "insert_100000_ms": 1386.7587500000127, + "wal_bytes": 38822584, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 6.914874999949916, + 6.668166999986624, + 6.523834000006445, + 6.556250000016917, + 6.811083000002327 + ], + "explain_execution_ms": 8.532 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 46.40233399999261, + 6.385249999993903, + 6.055750000030002, + 7.862708000004659, + 7.713500000022577 + ], + "explain_execution_ms": 7.042 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 6.462749999968764, + 6.372040999963247, + 6.377625000027365, + 6.360333999964496, + 6.293291999952544 + ], + "explain_execution_ms": 27.257 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.6799170000231243, + 2.6539589999856616, + 2.6237499999979264, + 2.6159999999890715, + 2.5921669999888763 + ], + "explain_execution_ms": 3.549 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.2941670000022896, + 2.2702500000377768, + 2.252708000014536, + 2.271042000018042, + 2.3105829999963134 + ], + "explain_execution_ms": 2.66 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.9595829999730086, + 1.9207080000001042, + 2.059125000016593, + 2.0110829999566704, + 2.0071669999879305 + ], + "explain_execution_ms": 1.827 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.3095409999550611, + 1.3007080000306814, + 1.288000000045031, + 1.280042000018966, + 1.2995419999697333 + ], + "explain_execution_ms": 1.592 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.156374999993659, + 1.1407089999693198, + 1.144167000006746, + 1.1433750000264808, + 1.1405409999838412 + ], + "explain_execution_ms": 1.222 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 1.0064999999599422, + 1.003166999964833, + 1.002000000028147, + 0.9964999999851898, + 0.9997909999697185 + ], + "explain_execution_ms": 0.846 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.3318340000078024, + 0.34162500003276364, + 0.3277920000073209, + 0.32820799998489747, + 0.33399999995253893 + ], + "explain_execution_ms": 0.204 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.31758300002593387, + 0.31320800002276883, + 0.3052920000072845, + 0.32208299995772904, + 0.31487500001503577 + ], + "explain_execution_ms": 0.151 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.29320900000584516, + 0.29254200001105346, + 0.30541699999275806, + 0.2934999999979482, + 0.2922080000189453 + ], + "explain_execution_ms": 0.106 + } + ] + }, + { + "variant": "history", + "repeat": 1, + "insert_100000_ms": 1404.4910420000178, + "wal_bytes": 43108512, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.3173750000087239, + 0.29504200000474157, + 0.28329200000598576, + 0.3037079999899106, + 0.31616600000461403 + ], + "explain_execution_ms": 0.021 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.2926249999859465, + 0.21762500000477303, + 0.19337500003757668, + 0.21075000000791988, + 0.23008300001947646 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.2326670000343256, + 0.22991600002342238, + 0.25349999998525163, + 0.2272909999874173, + 0.2270420000058948 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.2562920000173108, + 0.23475000000416912, + 0.22645799998599614, + 0.22308400002657436, + 0.2378749999820684 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.22491599997920275, + 0.25087500000609, + 0.22708300002705073, + 0.2274999999940519, + 0.22325000003320383 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.2279589999716336, + 0.22745899997289598, + 0.22599999999783904, + 0.2297500000167929, + 0.22641599997541562 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.24199999995744292, + 0.2252499999713109, + 0.22495799998978327, + 0.22799999999278953, + 0.2394169999888618 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.2292909999823678, + 0.3384169999662845, + 0.28125000000045475, + 0.24829200003750884, + 0.23558299994874687 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.2306659999931071, + 0.2268749999529973, + 0.29704099995342403, + 0.24354200002107973, + 0.23233300004221746 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.2479999999991378, + 0.2283750000060536, + 0.22741699996231546, + 0.2274999999940519, + 0.24287499996944462 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.22462499998709973, + 0.24941599997418962, + 0.2247919999831538, + 0.22466699999768025, + 0.2243339999949967 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.22287500001993976, + 0.2249590000360513, + 0.22291699997367687, + 0.2342500000054315, + 0.23004100000889594 + ], + "explain_execution_ms": 0.017 + } + ] + }, + { + "variant": "history", + "repeat": 2, + "insert_100000_ms": 1417.234876000009, + "wal_bytes": 43108512, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.3070409999850199, + 0.2651249999985339, + 0.2804580000201895, + 0.2466660000095544, + 0.4540420000012091 + ], + "explain_execution_ms": 0.021 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.30112500002132947, + 0.23020800000495, + 0.24920800001382304, + 0.26024999999663123, + 0.23162500002626984 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.2297500000167929, + 0.22941700001410936, + 0.2268339999886848, + 0.2509999999915635, + 0.2265840000177377 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.24954200000593119, + 0.22820899999942412, + 0.22675000002436718, + 0.22470900000826077, + 0.22833299999547307 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.2232080000226233, + 0.22866699998758122, + 0.23858299999801602, + 0.22420799996325513, + 0.22699999999531428 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.2425420000236045, + 0.22604199995157614, + 0.22554200000968194, + 0.22283300000935924, + 0.24158299999044175 + ], + "explain_execution_ms": 0.025 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.23279099997353114, + 0.23029199996926764, + 0.32120800000257077, + 0.26729199998953845, + 0.2517920000286722 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.23408300000937743, + 0.2288339999836353, + 0.23687499998459316, + 0.2249590000360513, + 0.2247919999831538 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.2243749999593092, + 0.22408299997778158, + 0.22312500004773028, + 0.23924999999280772, + 0.22150000000920045 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.22354100002530686, + 0.2302089999943746, + 0.22079199999325283, + 0.2222910000000411, + 0.2202919999945152 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.22287500001993976, + 0.22287499996309634, + 0.22041700003683218, + 0.2310830000169517, + 0.22616600000446851 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.22241599998551465, + 0.22308299998030634, + 0.22195799999735755, + 0.21995800000240706, + 0.21950000001424996 + ], + "explain_execution_ms": 0.017 + } + ] + }, + { + "variant": "chat_only", + "repeat": 2, + "insert_100000_ms": 1390.132249999965, + "wal_bytes": 38822560, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 7.372583000005761, + 6.758875000002718, + 6.808458000023165, + 6.906250000042746, + 7.094250000022839 + ], + "explain_execution_ms": 40.541 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 6.415666999998848, + 6.257875000017066, + 6.178666999971938, + 8.023291000029076, + 7.650417000036214 + ], + "explain_execution_ms": 7.198 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 42.71466599999485, + 6.7938749999711945, + 6.601541000009092, + 6.471875000045202, + 6.461500000000342 + ], + "explain_execution_ms": 5.685 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.8667090000453754, + 2.8354170000284284, + 2.8501250000090295, + 2.787916999977824, + 2.8024999999729516 + ], + "explain_execution_ms": 3.787 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.3539160000041193, + 2.3988750000398795, + 2.3642079999603993, + 2.349291000030007, + 2.33629199999541 + ], + "explain_execution_ms": 2.82 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.9656659999895965, + 1.9535420000238446, + 1.9440419999909864, + 1.940166999986559, + 1.9436249999671418 + ], + "explain_execution_ms": 1.83 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.3667079999777343, + 1.36754199996858, + 1.3556669999843507, + 1.3793749999990723, + 1.357499999983247 + ], + "explain_execution_ms": 1.715 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.1639999999601969, + 1.1638330000209862, + 1.1650000000145155, + 1.1809580000203823, + 1.1649159999933545 + ], + "explain_execution_ms": 1.275 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.9753749999958927, + 0.9995409999987714, + 1.014125000040167, + 0.9871670000052291, + 0.9951249999744505 + ], + "explain_execution_ms": 0.854 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.3327910000052725, + 0.3331660000185366, + 0.3437500000131877, + 0.3285419999770056, + 0.3382080000164933 + ], + "explain_execution_ms": 0.22 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.3201250000302025, + 0.33404200001996287, + 0.31708299997035283, + 0.31666699999277625, + 0.3300420000300619 + ], + "explain_execution_ms": 0.156 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.3005840000014359, + 0.28595899999572794, + 0.2849999999625652, + 0.2990409999483745, + 0.2847920000021986 + ], + "explain_execution_ms": 0.106 + } + ] + }, + { + "variant": "chat_only", + "repeat": 3, + "insert_100000_ms": 1330.3022509999778, + "wal_bytes": 38822560, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 48.38283300000512, + 7.059540999989622, + 6.774750000033691, + 7.226040999967154, + 7.049875000006978 + ], + "explain_execution_ms": 8.497 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 5.816499999980351, + 5.940875000021606, + 52.47504099997968, + 7.940541999971629, + 7.566457999985232 + ], + "explain_execution_ms": 6.953 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 6.405041999983041, + 6.38737500003117, + 6.312415999957466, + 6.295207999983177, + 6.2736250000057225 + ], + "explain_execution_ms": 5.548 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.6689579999583657, + 2.6040839999836862, + 2.6041659999691547, + 2.593499999989035, + 2.596417000006568 + ], + "explain_execution_ms": 11.285 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.3674579999806156, + 2.497582999978931, + 2.4268750000260297, + 2.543584000022747, + 2.3092499999961547 + ], + "explain_execution_ms": 2.655 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.982666000003519, + 1.9526250000012624, + 1.9421659999920848, + 2.0057080000128735, + 1.990875000046799 + ], + "explain_execution_ms": 1.878 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.4241659999925105, + 1.449707999995553, + 1.3514999999983957, + 1.3168329999757589, + 1.3098750000040127 + ], + "explain_execution_ms": 1.607 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.1759579999761627, + 1.1762500000145337, + 1.1795839999990676, + 1.1871670000118684, + 1.1549170000080267 + ], + "explain_execution_ms": 1.221 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.9988330000396672, + 1.0168339999836462, + 1.0056670000153645, + 0.9815419999767983, + 0.9922919999780788 + ], + "explain_execution_ms": 0.831 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.33158299999058727, + 0.32433300003731347, + 0.3312079999773232, + 0.3266250000137916, + 0.3368750000163345 + ], + "explain_execution_ms": 0.194 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.3318330000183778, + 0.3219579999722555, + 0.3167920000350932, + 0.317624999979671, + 0.3145420000123522 + ], + "explain_execution_ms": 0.154 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.29337500001247463, + 0.29699999998911153, + 0.3068749999783904, + 0.309750000042186, + 0.29841700001043137 + ], + "explain_execution_ms": 0.107 + } + ] + }, + { + "variant": "history", + "repeat": 3, + "insert_100000_ms": 1366.9856260000302, + "wal_bytes": 43108536, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.29145799999241717, + 0.25487500005283437, + 0.24249999995618055, + 0.24029200000086348, + 0.40070800002922624 + ], + "explain_execution_ms": 0.02 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.24608299997908034, + 0.2166659999716103, + 0.19137499998578278, + 0.20683299999291194, + 0.18729199996414536 + ], + "explain_execution_ms": 0.034 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.1954580000074202, + 0.19033400002399503, + 0.1846659999955591, + 0.20504200000459605, + 0.1987920000487975 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.1877920000197264, + 0.2019580000478527, + 0.1839579999796115, + 0.18112499998323983, + 0.18541700001151185 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.19970799996826827, + 0.18504199999824777, + 0.18249999999397915, + 0.17937500001607987, + 0.1814999999965039 + ], + "explain_execution_ms": 0.016 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.18754100000251128, + 0.18725000001040826, + 0.18091699996602983, + 0.22283300000935924, + 0.20837499999970532 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.1893749999908323, + 0.20050000000537693, + 0.18170799995687048, + 0.1799160000359734, + 0.19962499999337524 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.2502920000324593, + 0.22045800000114468, + 0.21791599999687605, + 0.2180409999823496, + 0.23691699999517368 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.21720799998092843, + 0.21795899999688118, + 0.21658399998614186, + 0.2168750000350883, + 0.21812500000351065 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.21791700004314407, + 0.2171250000060354, + 0.21858299999166775, + 0.21612500000856016, + 0.22783299999673545 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.2180000000180371, + 0.21766600002592895, + 0.21808299999293013, + 0.2169169999888254, + 0.2186660000234042 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.21783400001140762, + 0.21633299996892674, + 0.2234579999935704, + 0.21850000001677472, + 0.2274999999940519 + ], + "explain_execution_ms": 0.017 + } + ] + } + ], + "snapshot": { + "counts": { + "users": 10, + "chats": 4, + "chat_members": 8, + "messages": 100000, + "schema_migrations": 1 + }, + "messages_sha256": "36958977cf7f00bc537cb8e2b0052fd83cc8358d0d44d72304d0856d1cf43870", + "samples": [ + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.000001+00:00", + "id": "0406791b-bf11-c4be-113d-e56c41008a52", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.035000+00:00", + "id": "169ee7a7-ca0d-aa05-bd8c-a1eb275708e7", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 35000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.070000+00:00", + "id": "dbf177a7-e559-4eb5-2f16-d7594a0c375b", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 70000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.070001+00:00", + "id": "042eb2e1-c673-28b2-57e6-bbc531f9f886", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 70001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.080000+00:00", + "id": "9042bcb2-df86-667b-9e54-e6e6c0c7923a", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 80000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.090000+00:00", + "id": "5409a1ee-b7e3-f2fa-1b87-173c59924f20", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 90000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.090001+00:00", + "id": "8a98c150-4c98-8f1a-478f-2d01e8725584", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 90001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.094500+00:00", + "id": "64cc6094-35b0-52bc-2c33-655d810cbabc", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 94500 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.099000+00:00", + "id": "26f56eb2-d617-f971-efa9-fc332ae1272b", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 99000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.099001+00:00", + "id": "7d565f77-f911-6eb9-62a9-dcf414afb5f3", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 99001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.099500+00:00", + "id": "86263544-3ca9-88ac-6a18-01fe3ed3d579", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 99500 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.100000+00:00", + "id": "103bbef2-c217-19c4-d209-a1c1cdd3a1e2", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 100000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + } + ] + }, + "utc": "2026-10-22T11:09:43.745407+00:00", + "project": "lab2-research-85021a09d9", + "command": "./scripts/research.sh", + "host": "Darwin arm64", + "commit": "9961362 + working tree", + "postgres_limits": { + "cpus": 1, + "memory_mib": 512 + }, + "api_runner_limits": { + "cpus": 1, + "memory_mib": 256 + }, + "source": { + "container": "c0b5e13dc5d2fd8bbd488a9b227eb62e4fa6a3896f6b28cf464b94e3d8f9032f", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "volumes": [ + "lab2-research-85021a09d9_postgres_data" + ] + }, + "restored": { + "container": "1c23bc29d590048481d8e2d18dfe340627309f20f004d4ffdb8892c3582fb8f1", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "volumes": [ + "lab2-research-85021a09d9_restore_data" + ] + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "stats_after_queries": "{\"BlockIO\":\"131kB / 583MB\",\"CPUPerc\":\"0.05%\",\"Container\":\"c0b5e13dc5d2fd8bbd488a9b227eb62e4fa6a3896f6b28cf464b94e3d8f9032f\",\"ID\":\"c0b5e13dc5d2\",\"MemPerc\":\"12.35%\",\"MemUsage\":\"63.24MiB / 512MiB\",\"Name\":\"lab2-research-85021a09d9-postgres-1\",\"NetIO\":\"378kB / 33.3MB\",\"PIDs\":\"6\"}", + "backup": { + "file": "backups/lab2-research-85021a09d9.dump", + "bytes": 3067615, + "seconds": 0.5438075830461457 + }, + "restore": { + "seconds": 0.6752373329363763, + "verification_seconds": 2.342134208069183, + "verification": { + "counts": { + "users": 10, + "chats": 4, + "chat_members": 8, + "messages": 100000, + "schema_migrations": 1 + }, + "messages_sha256": "36958977cf7f00bc537cb8e2b0052fd83cc8358d0d44d72304d0856d1cf43870", + "samples_verified": 12, + "all_messages_match": true, + "after_backup_message_missing_in_restore": true, + "source_messages": 100001 + }, + "post_backup_insert": { + "id": "9b10784e-e462-01b4-6520-1c572ae115d9", + "source_messages": 100001 + } + }, + "total_seconds": 22.895743250031956 +} \ No newline at end of file diff --git a/evidence/research.txt b/evidence/research.txt new file mode 100644 index 00000000..ef6bb5e7 --- /dev/null +++ b/evidence/research.txt @@ -0,0 +1,32 @@ +Research project: lab2-research-85021a09d9 +Measuring two indexes on the same 100000 messages, three repeats... +{ + "chat_only": { + "insert_100000_median_ms": 1386.7587500000127, + "index_bytes": 671744, + "wal_bytes_median": 38822560, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 1.6852079999978287, + "p95_ms": 7.566457999985232, + "p99_ms": 46.40233399999261, + "successful_sql_rps": 282.01364182715855 + }, + "history": { + "insert_100000_median_ms": 1404.4910420000178, + "index_bytes": 7315456, + "wal_bytes_median": 43108512, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 0.22610399997802233, + "p95_ms": 0.29704099995342403, + "p99_ms": 0.3384169999662845, + "successful_sql_rps": 4322.810483886352 + } +} +PASS: backup 0.54s; restore 0.68s; 100000 messages, all hashes and 12 samples match +PASS: restored into a different container and fresh volume; one post-backup commit is absent as expected +Results: evidence/research.json, evidence/query-plans.json; dump stays outside Git in backups/ +Removed only temporary research project: lab2-research-85021a09d9 diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..c39b8d54 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,40 @@ +global + log stdout format raw local0 + maxconn 256 + +defaults + mode http + log global + option dontlognull + option http-server-close + timeout connect 1s + timeout client 40s + timeout server 60s + timeout http-request 5s + timeout check 1s + retries 1 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + hold other 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server inter 1s fastinter 1s downinter 1s fall 2 rise 2 + server-template replica 10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..9dfbb863 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (char_length(display_name) >= 1), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (char_length(title) >= 1), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000 AND text !~ '^\s*$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_message_idempotency.sql b/migrations/002_message_idempotency.sql new file mode 100644 index 00000000..375b58c4 --- /dev/null +++ b/migrations/002_message_idempotency.sql @@ -0,0 +1,10 @@ +-- Добавочная миграция: существующие данные и таблицы lab2 сохраняются. +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_text text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..5a9db48e --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,34 @@ +-- Только добавления к lab3; старые сообщения и ключи сохраняются. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL, + object_key text NOT NULL UNIQUE, + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); +ALTER TABLE message_idempotency + ADD COLUMN request_attachment_ids uuid[] NOT NULL DEFAULT '{}'; + +-- Фиксируется ДО S3 PUT. После COMMIT ready-метаданных удаляется атомарно с ними. +CREATE TABLE attachment_uploads ( + id uuid PRIMARY KEY, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachment_uploads_created_idx ON attachment_uploads(created_at); diff --git a/migrations/004_attachment_jobs.sql b/migrations/004_attachment_jobs.sql new file mode 100644 index 00000000..dd53682d --- /dev/null +++ b/migrations/004_attachment_jobs.sql @@ -0,0 +1,42 @@ +-- Lab5: существующие ready-вложения lab4 остаются доступными. +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')), + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN error_code text, + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); +UPDATE attachments SET updated_at=created_at; + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1), + staging_key text NOT NULL UNIQUE, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt>=1), + max_attempts integer NOT NULL CHECK (max_attempts>=1), + replay_count integer NOT NULL DEFAULT 0, + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_error text, + staging_deleted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(lease_until) WHERE lease_until IS NOT NULL; + +CREATE TABLE attachment_outbox ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + PRIMARY KEY(job_id, attempt) +); +CREATE INDEX attachment_outbox_due_idx ON attachment_outbox(available_at) WHERE published_at IS NULL; + +CREATE TABLE attachment_job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + event text NOT NULL, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2bb42d3b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,36 @@ +# Основные библиотеки +fastapi==0.142.4 +uvicorn==0.54.0 +pydantic==2.13.5 + +# Их зависимости +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +pydantic-core==2.46.5 +starlette==1.7.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 + +# PostgreSQL, сессии и Argon2id +psycopg[binary,pool]==3.3.6 +redis==8.1.0 +argon2-cffi==25.1.0 + +# Транзитивные зависимости новых библиотек +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +pycparser==3.11 + +# S3 и проверка настоящих PNG/JPEG (без изменения оригинала) +minio==7.2.20 +Pillow==13.0.0 +certifi==2026.10.23 +pycryptodome==4.3.0 +urllib3==2.8.0 diff --git a/research/README.md b/research/README.md new file mode 100644 index 00000000..956721c4 --- /dev/null +++ b/research/README.md @@ -0,0 +1,12 @@ +# Трек 2 на оценку 5 + +Запуск из корня: `./scripts/research.sh`. Нужны Docker Compose, Python 3 и интернет для первой сборки. Скрипт собирает API-образ и использует отдельный Compose project с новыми volumes. Текущий стенд не участвует. + +- run.py управляет контейнерами, pg_dump, pg_restore, файлами evidence и cleanup. +- benchmark.py выполняет SQL и сверку внутри API-образа. Перед заполнением требует пустую БД. Исследовательский флаг устанавливает только драйвер. + +Датасет: 100000 сообщений в 4 чатах (70000/20000/9000/1000). Сравниваются индекс chat_id и индекс (chat_id,created_at,id), по 3 повтора с одинаковыми ограничениями и записями. На каждый чат — страницы около 10/50/90%, один прогрев и пять измерений, 51 строка. Измеряется INSERT SELECT с COMMIT, WAL и размер индекса. EXPLAIN и основные цифры сохраняются в evidence. + +После dump создаётся ещё одно сообщение. Backup восстанавливается в другой контейнер и свежий volume. Сравниваются все счётчики, 12 полных записей и SHA-256 всех сообщений. Последующая запись ожидаемо отсутствует в восстановленном снимке. + +Файлы research.json, query-plans.json и research.txt обновляются автоматически. Dump остаётся в игнорируемом backups/. Временные исследовательские volumes удаляются после проверки. Методика, фактические результаты и ограничения лабораторной 2 описаны в [REPORT.lab2.md](../REPORT.lab2.md). diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/research/benchmark.py b/research/benchmark.py new file mode 100644 index 00000000..93b51fe4 --- /dev/null +++ b/research/benchmark.py @@ -0,0 +1,168 @@ +"""SQL-эксперимент запускается только в отдельном research Compose project.""" +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +import secrets +import statistics +import sys +import time +from uuid import UUID + +import psycopg +from psycopg.conninfo import make_conninfo +from psycopg.rows import dict_row + +from src import db, migrate, passwords +from src.config import secret + + +SIZES = [70000, 20000, 9000, 1000] +START = datetime(2026, 1, 1, tzinfo=timezone.utc) +QUERY = """SELECT * FROM messages WHERE chat_id=%s AND (created_at, id)>(%s, %s) + ORDER BY created_at, id LIMIT %s""" + + +def identifier(value): + return UUID(hashlib.md5(value.encode(), usedforsecurity=False).hexdigest()) + + +def connect(restored=False): + url = secret("DATABASE_URL") + if restored: + url = make_conninfo(url, host="postgres-restore") + return psycopg.connect(url, autocommit=True, row_factory=dict_row, + options="-c timezone=UTC -c statement_timeout=120000") + + +def snapshot(conn): + counts = {} + for table in ("users", "chats", "chat_members", "messages", "schema_migrations"): + # Имена таблиц — фиксированный список выше, не входные данные клиента. + counts[table] = conn.execute("SELECT count(*) AS n FROM " + table).fetchone()["n"] + digest = hashlib.sha256() + for row in conn.execute("SELECT * FROM messages ORDER BY id"): + digest.update((json.dumps(row, default=str, sort_keys=True, ensure_ascii=False) + "\n").encode()) + samples = [] + offset = 0 + for size in SIZES: + for number in (offset + 1, offset + size // 2, offset + size): + row = conn.execute("SELECT * FROM messages WHERE id=%s", (identifier(f"message:{number}"),)).fetchone() + samples.append(json.loads(json.dumps(row, default=str, sort_keys=True, ensure_ascii=False))) + offset += size + return {"counts": counts, "messages_sha256": digest.hexdigest(), "samples": samples} + + +def benchmark(): + db.pool.open(wait=True, timeout=15) + try: + migrate.apply() + finally: + db.pool.close() + results, plans = [], [] + with connect() as conn: + assert conn.execute("SELECT count(*) AS n FROM users").fetchone()["n"] == 0, "Research requires an empty database" + encoded = passwords.hash_password(secrets.token_urlsafe(24)) + for n in range(1, 11): + conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s)", + (identifier(f"user:{n}"), f"research_{n}", "Synthetic research user", encoded)) + for n in range(4): + chat = identifier(f"chat:{n}") + conn.execute("INSERT INTO chats(id,title) VALUES(%s,%s)", (chat, f"Research chat {n}")) + for user in (1, 2): + conn.execute("INSERT INTO chat_members(chat_id,user_id) VALUES(%s,%s)", (chat, identifier(f"user:{user}"))) + variants = [("chat_only", "chat_id"), ("history", "chat_id, created_at, id")] + for repeat in range(3): + order = variants if repeat % 2 == 0 else variants[::-1] + for variant, columns in order: + conn.execute("TRUNCATE messages") + conn.execute("DROP INDEX IF EXISTS messages_history_idx") + conn.execute("DROP INDEX IF EXISTS messages_chat_idx") + index = "messages_history_idx" if variant == "history" else "messages_chat_idx" + conn.execute("CREATE INDEX " + index + " ON messages (" + columns + ")") + lsn = conn.execute("SELECT pg_current_wal_insert_lsn() AS lsn").fetchone()["lsn"] + started = time.perf_counter() + conn.execute("""INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT md5('message:'||n)::uuid, + md5('chat:'||(CASE WHEN n<=70000 THEN 0 WHEN n<=90000 THEN 1 WHEN n<=99000 THEN 2 ELSE 3 END))::uuid, + md5('user:'||(1+n%2))::uuid, + 'Synthetic message '||n||' '||repeat('x',80), + timestamptz '2026-01-01 00:00:00+00' + n*interval '1 microsecond' + FROM generate_series(1,100000) AS n""") + insert_ms = (time.perf_counter() - started) * 1000 + wal = conn.execute("SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(), %s) AS bytes", (lsn,)).fetchone()["bytes"] + conn.execute("VACUUM (ANALYZE) messages") + index_bytes = conn.execute("SELECT pg_relation_size(%s::regclass) AS bytes", (index,)).fetchone()["bytes"] + result = {"variant": variant, "repeat": repeat + 1, "insert_100000_ms": insert_ms, + "wal_bytes": int(wal), "index_bytes": index_bytes, "queries": []} + offset = 0 + for chat, size in enumerate(SIZES): + for fraction in (0.1, 0.5, 0.9): + number = offset + int(size * fraction) + params = (identifier(f"chat:{chat}"), START + timedelta(microseconds=number), identifier(f"message:{number}"), 51) + conn.execute(QUERY, params).fetchall() # один прогрев именно этой страницы + times = [] + for _ in range(5): + started = time.perf_counter() + rows = conn.execute(QUERY, params).fetchall() + times.append((time.perf_counter() - started) * 1000) + assert len(rows) == 51 + plan = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + QUERY, params).fetchone()["QUERY PLAN"][0] + result["queries"].append({"chat_size": size, "position": fraction, "latency_ms": times, + "explain_execution_ms": plan["Execution Time"]}) + if repeat == 0: + plans.append({"variant": variant, "chat_size": size, "position": fraction, "plan": plan}) + offset += size + results.append(result) + # Последняя итерация заканчивается production-индексом history. + assert conn.execute("SELECT to_regclass('messages_history_idx') AS name").fetchone()["name"] + baseline = snapshot(conn) + assert baseline["counts"]["messages"] == 100000 + summary = {} + for variant, _ in variants: + trials = [row for row in results if row["variant"] == variant] + latencies = sorted(t for row in trials for q in row["queries"] for t in q["latency_ms"]) + summary[variant] = { + "insert_100000_median_ms": statistics.median(row["insert_100000_ms"] for row in trials), + "index_bytes": trials[-1]["index_bytes"], + "wal_bytes_median": statistics.median(row["wal_bytes"] for row in trials), + "queries": len(latencies), "errors": 0, "concurrency": 1, + "p50_ms": statistics.median(latencies), + "p95_ms": latencies[int(0.95 * (len(latencies) - 1))], + "p99_ms": latencies[int(0.99 * (len(latencies) - 1))], + "successful_sql_rps": len(latencies) / (sum(latencies) / 1000), + } + return {"postgres": conn.execute("SELECT version() AS version").fetchone()["version"], + "messages": 100000, "chat_sizes": SIZES, "repeats": 3, "warmup_per_case": 1, + "measurements_per_case": 5, "page_size": 51, "query": QUERY, + "summary": summary, "trials": results, "plans": plans, "snapshot": baseline} + + +def after_backup(): + with connect() as conn: + message_id = identifier("after-backup") + conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES(%s,%s,%s,%s)", + (message_id, identifier("chat:0"), identifier("user:1"), "Committed AFTER backup")) + return {"id": str(message_id), "source_messages": conn.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]} + + +def verify(): + expected = json.load(sys.stdin) + with connect(restored=True) as restored, connect() as source: + actual = snapshot(restored) + assert actual == expected, "Restored data differs from the snapshot" + marker = (identifier("after-backup"),) + assert restored.execute("SELECT 1 FROM messages WHERE id=%s", marker).fetchone() is None + assert source.execute("SELECT 1 FROM messages WHERE id=%s", marker).fetchone() is not None + return {"counts": actual["counts"], "messages_sha256": actual["messages_sha256"], + "samples_verified": len(actual["samples"]), "all_messages_match": True, + "after_backup_message_missing_in_restore": True, + "source_messages": source.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]} + + +if __name__ == "__main__": + if os.environ.get("LAB2_RESEARCH") != "1": + raise SystemExit("Use ./scripts/research.sh: an isolated research database is required") + operation = sys.argv[1] if len(sys.argv) > 1 else "benchmark" + result = {"benchmark": benchmark, "after-backup": after_backup, "verify": verify}[operation]() + print(json.dumps(result, ensure_ascii=False, default=str)) diff --git a/research/run.py b/research/run.py new file mode 100644 index 00000000..33e0f0e3 --- /dev/null +++ b/research/run.py @@ -0,0 +1,99 @@ +"""Создаёт изолированный стенд, измеряет SQL, делает dump и restore; пишет реальные результаты.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +project = "lab2-research-" + uuid4().hex[:10] +env = dict(os.environ, COMPOSE_PROJECT_NAME=project) +command = ["docker", "compose", "--profile", "research"] + + +def compose(*args, stdin=None, stdout=subprocess.PIPE): + return subprocess.run([*command, *args], cwd=ROOT, env=env, input=stdin, + stdout=stdout, stderr=subprocess.PIPE, check=True, timeout=600) + + +def run_module(operation, stdin=None): + result = compose("run", "--rm", "--no-deps", "-T", "-e", "LAB2_RESEARCH=1", "api", "python", "-m", "research.benchmark", operation, stdin=stdin) + return json.loads(result.stdout) + + +def wait_database(service): + for _ in range(60): + cid = compose("ps", "-q", service).stdout.decode().strip() + state = subprocess.run(["docker", "inspect", "--format", "{{.State.Health.Status}}", cid], capture_output=True, text=True, check=True) + if state.stdout.strip() == "healthy": + return + time.sleep(1) + raise RuntimeError(service + " did not become healthy") + + +def main(): + evidence = ROOT / "evidence" + evidence.mkdir(exist_ok=True) + backups = ROOT / "backups" + backups.mkdir(exist_ok=True) + dump = backups / (project + ".dump") + metadata = {"utc": datetime.now(timezone.utc).isoformat(), "project": project, + "command": "./scripts/research.sh", "host": platform.system() + " " + platform.machine(), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() + " + working tree", + "postgres_limits": {"cpus": 1, "memory_mib": 512}, "api_runner_limits": {"cpus": 1, "memory_mib": 256}} + started = time.perf_counter() + try: + print("Research project:", project, flush=True) + compose("up", "-d", "postgres", "postgres-restore") + wait_database("postgres") + wait_database("postgres-restore") + print("Measuring two indexes on the same 100000 messages, three repeats...", flush=True) + result = run_module("benchmark") + plans = result.pop("plans") + (evidence / "query-plans.json").write_text(json.dumps(plans, ensure_ascii=False, indent=2)) + source_id = compose("ps", "-q", "postgres").stdout.decode().strip() + restore_id = compose("ps", "-q", "postgres-restore").stdout.decode().strip() + assert source_id != restore_id + for label, cid in (("source", source_id), ("restored", restore_id)): + inspected = json.loads(subprocess.run(["docker", "inspect", cid], capture_output=True, check=True).stdout)[0] + metadata[label] = {"container": cid, "image": inspected["Image"], + "volumes": [m["Name"] for m in inspected["Mounts"] if m["Type"] == "volume"]} + assert metadata["source"]["volumes"] != metadata["restored"]["volumes"] + metadata["docker"] = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"], capture_output=True, text=True, check=True).stdout.strip() + metadata["stats_after_queries"] = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", source_id], capture_output=True, text=True, check=True).stdout.strip() + backup_started = time.perf_counter() + with dump.open("wb") as output: + compose("exec", "-T", "postgres", "pg_dump", "-U", "messenger", "-d", "messenger", "--format=custom", stdout=output) + backup_seconds = time.perf_counter() - backup_started + marker = run_module("after-backup") + restore_started = time.perf_counter() + # Dump подаётся через stdin, без вывода данных или credentials в журнал. + with dump.open("rb") as input_file: + subprocess.run([*command, "exec", "-T", "postgres-restore", "pg_restore", "-U", "messenger", "-d", "messenger", + "--exit-on-error", "--single-transaction", "--no-owner"], cwd=ROOT, env=env, stdin=input_file, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True, timeout=300) + restore_seconds = time.perf_counter() - restore_started + verification_started = time.perf_counter() + verification = run_module("verify", stdin=json.dumps(result["snapshot"]).encode()) + result.update(metadata) + result["backup"] = {"file": "backups/" + dump.name, "bytes": dump.stat().st_size, "seconds": backup_seconds} + result["restore"] = {"seconds": restore_seconds, "verification_seconds": time.perf_counter() - verification_started, + "verification": verification, "post_backup_insert": marker} + result["total_seconds"] = time.perf_counter() - started + (evidence / "research.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + print(json.dumps(result["summary"], ensure_ascii=False, indent=2), flush=True) + print(f"PASS: backup {backup_seconds:.2f}s; restore {restore_seconds:.2f}s; 100000 messages, all hashes and 12 samples match", flush=True) + print("PASS: restored into a different container and fresh volume; one post-backup commit is absent as expected", flush=True) + print("Results: evidence/research.json, evidence/query-plans.json; dump stays outside Git in backups/", flush=True) + finally: + compose("down", "--volumes", "--timeout", "10") + print("Removed only temporary research project:", project, flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000..0a0b9999 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +python3 scripts/init_local.py +docker compose build "$@" api diff --git a/scripts/check-docker.sh b/scripts/check-docker.sh new file mode 100755 index 00000000..df5799c4 --- /dev/null +++ b/scripts/check-docker.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +if [[ ! -x .venv/bin/python ]]; then + echo "Создай .venv и установи requirements.txt по REPORT.md" >&2 + exit 1 +fi +./scripts/build.sh +mkdir -p evidence +.venv/bin/python -u student-tests/lab5_pipeline.py 2>&1 | sed -E 's/[[:blank:]]+$//' | tee evidence/lab5-checks.txt diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100755 index 00000000..90a608d1 --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" + +if [[ ! -x .venv/bin/python ]]; then + echo "Не хватает .venv" >&2 + echo "python3 -m venv .venv" >&2 + echo ".venv/bin/python -m pip install -r requirements.txt" >&2 + exit 1 +fi + +export BASE_URL="${BASE_URL:-http://localhost:8080}" +echo "Проверяем сервер: $BASE_URL" +make check test PYTHON="$project_dir/.venv/bin/python" BASE_URL="$BASE_URL" +.venv/bin/python -m unittest discover -s student-tests -v diff --git a/scripts/demo.sh b/scripts/demo.sh new file mode 100755 index 00000000..af83a80a --- /dev/null +++ b/scripts/demo.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +base_url="${BASE_URL:-http://localhost:8080}" +suffix="$(date +%s)_$RANDOM" +# Только синтетические пароли; токены остаются в памяти shell и не выводятся. +password="Demo-password-$suffix" +register() { + local username="$1" display_name="$2" + jq -nc --arg u "$username" --arg d "$display_name" --arg p "$password" \ + '{username:$u, display_name:$d, password:$p}' | + curl -fsS -H 'Content-Type: application/json' --data-binary @- "$base_url/api/v1/users" | jq -er '.id' +} +login() { + curl -fsS -X POST -u "$1:$password" "$base_url/api/v1/auth/sessions" | jq -er '.token' +} +alice_id="$(register "alice_$suffix" Алиса)" +bob_id="$(register "bob_$suffix" Боб)" +alice_token="$(login "alice_$suffix")" +bob_token="$(login "bob_$suffix")" +chat_id="$(jq -nc --arg a "$alice_id" --arg b "$bob_id" '{title:"Учебный чат",member_ids:[$a,$b]}' | + curl -fsS -H "Authorization: Bearer $alice_token" -H 'Content-Type: application/json' \ + --data-binary @- "$base_url/api/v1/chats" | jq -er '.id')" +for token in "$alice_token" "$bob_token"; do + curl -fsS -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \ + -H "Idempotency-Key: demo-$RANDOM-$RANDOM" \ + -d '{"text":"Привет из lab5! 👋"}' "$base_url/api/v1/chats/$chat_id/messages" | jq . +done +curl -fsS -H "Authorization: Bearer $bob_token" "$base_url/api/v1/chats/$chat_id/messages" | jq . diff --git a/scripts/init_local.py b/scripts/init_local.py new file mode 100644 index 00000000..64b36789 --- /dev/null +++ b/scripts/init_local.py @@ -0,0 +1,62 @@ +"""Создаёт локальное окружение и secrets; существующие значения сохраняет.""" +from pathlib import Path +import json +import os +import re +import secrets +import shutil +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +if not (root / ".env").exists(): + shutil.copyfile(root / ".env.example", root / ".env") +directory = root / "secrets" +directory.mkdir(exist_ok=True, mode=0o700) +directory.chmod(0o700) +password_file = directory / "postgres_password" +if not password_file.exists(): + password_file.write_text(secrets.token_hex(24)) +password = password_file.read_text().strip() +values = { + "database_url": f"postgresql://messenger:{quote(password, safe='')}@postgres:5432/messenger", + "cache_url": "redis://valkey:6379/0", + "broker_password": secrets.token_hex(24), + "s3_root_user": "root-" + secrets.token_hex(8), + "s3_root_password": secrets.token_urlsafe(32), + "s3_access_key_id": "api-" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(32), +} +for name, value in values.items(): + path = directory / name + if not path.exists(): + path.write_text(value) +broker_url = directory / "broker_url" +if not broker_url.exists(): + broker_url.write_text("redis://:" + (directory / "broker_password").read_text().strip() + "@broker:6379/0") +broker_url.chmod(0o444) +dotenv = dict(line.split("=", 1) for line in (root / ".env").read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") and "=" in line) +bucket = os.environ.get("S3_BUCKET", dotenv.get("S3_BUCKET", "messenger-attachments")) +if re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket) is None: + raise ValueError("Недопустимое имя S3 bucket") +def local_secret(name): + return (directory / name).read_text().strip() +config = {"accounts": [{"id": "messenger-api", "displayName": "messenger-api"}], "identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": local_secret("s3_root_user"), + "secretKey": local_secret("s3_root_password")}], "actions": ["Admin"]}, + {"name": "messenger-api", "account": {"id": "messenger-api", "displayName": "messenger-api"}, + "credentials": [{"accessKey": local_secret("s3_access_key_id"), "secretKey": local_secret("s3_secret_access_key")}], + "actions": ["Read:" + bucket, "List:" + bucket, "Write:" + bucket + "/attachments/*"]}, +]} +config_file = directory / "s3_config" +encoded_config = json.dumps(config) +if not config_file.exists() or config_file.read_text() != encoded_config: + temporary = directory / "s3_config.tmp" + temporary.write_text(encoded_config) + temporary.chmod(0o444) + temporary.replace(config_file) +# Каталог 0700 защищает файлы на хосте; 0444 позволяет USER 10001 читать bind-mounted secret. +for name in ("postgres_password", "s3_config", *values): + (directory / name).chmod(0o444) +print("Local .env and secrets are ready; values are not printed") diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 00000000..9f0bb213 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +./scripts/build.sh +docker compose run --rm migrate diff --git a/scripts/replicas.py b/scripts/replicas.py new file mode 100644 index 00000000..e9cfe3e6 --- /dev/null +++ b/scripts/replicas.py @@ -0,0 +1,90 @@ +"""Управление репликами только текущего Compose project через Docker CLI.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, timeout=120).strip() + + +def replicas(): + ids = command("docker", "compose", "ps", "-q", "api").split() + return json.loads(command("docker", "inspect", *ids)) if ids else [] + + +def base_url(): + config = json.loads(command("docker", "compose", "config", "--format", "json")) + port = config["services"]["lb"]["ports"][0]["published"] + return os.environ.get("BASE_URL", "http://127.0.0.1:" + str(port)).rstrip("/") + + +def wait_ready(): + url = base_url() + "/health/ready" + deadline = time.monotonic() + 90 + seen = set() + while time.monotonic() < deadline: + current = replicas() + expected = {info["Config"]["Hostname"] for info in current} + if expected and all(info["State"].get("Health", {}).get("Status") == "healthy" for info in current): + try: + with urlopen(url, timeout=2) as response: + if response.status == 200: + seen.add(response.headers.get("X-Instance-Id")) + if expected <= seen: + print("API ready through LB:", ", ".join(sorted(expected)), flush=True) + return + except OSError: + pass + time.sleep(0.2) + raise RuntimeError("Не все работающие API-реплики готовы через LB за 90 секунд") + + +def stop_one(): + target = os.environ.get("TARGET_INSTANCE_ID") + if not target: + raise RuntimeError("Задайте TARGET_INSTANCE_ID из X-Instance-Id") + current = replicas() + matches = [info for info in current if info["Config"]["Hostname"] == target] + if len(current) < 2 or len(matches) != 1: + raise RuntimeError("Нужны минимум две работающие реплики и ровно одна выбранная") + selected = matches[0] + command("docker", "stop", "--time", "8", selected["Id"]) + print("Stopped API:", target, "container=" + selected["Id"], flush=True) + + +def rolling(): + wait_ready() + originals = replicas() + count = len(originals) + if count < 2: + raise RuntimeError("Для поочерёдной замены нужны минимум две API-реплики") + for old in originals: + active = replicas() + healthy = [info for info in active if info["Id"] != old["Id"] + and info["State"].get("Health", {}).get("Status") == "healthy"] + if not healthy: + raise RuntimeError("Нет готовой реплики, которая продолжит обслуживать запросы") + print("Replacing API:", old["Config"]["Hostname"], flush=True) + command("docker", "stop", "--time", "8", old["Id"]) + command("docker", "rm", old["Id"]) + # --no-recreate сохраняет остальные процессы; Compose создаёт только недостающий. + command("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api") + wait_ready() + assert len(replicas()) == count + final = replicas() + assert not ({info["Id"] for info in originals} & {info["Id"] for info in final}) + print("PASS: all API containers replaced sequentially; stores and LB kept running", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["wait", "stop", "rolling"]) + action = parser.parse_args().action + {"wait": wait_ready, "stop": stop_one, "rolling": rolling}[action]() diff --git a/scripts/research.sh b/scripts/research.sh new file mode 100755 index 00000000..b4f8e34e --- /dev/null +++ b/scripts/research.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +./scripts/build.sh +mkdir -p evidence +python3 -u research/run.py 2>&1 | tee evidence/research.txt diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f2eab1b8 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose restart --timeout 8 postgres valkey s3-storage s3 broker api lb cleanup publisher worker +./scripts/wait-ready.sh diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..b5b225ff --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +# Сначала собираем образ и применяем только обратно совместимые миграции. +./scripts/migrate.sh +exec python3 scripts/replicas.py rolling diff --git a/scripts/run-local.sh b/scripts/run-local.sh new file mode 100755 index 00000000..1c0b0e7c --- /dev/null +++ b/scripts/run-local.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +# Локальный стенд: HAProxy, две API-реплики, PostgreSQL и Valkey. +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +exec "$project_dir/scripts/start.sh" diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 00000000..15b0c61a --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose start worker diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 00000000..6dadfe6f --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +python3 scripts/init_local.py +docker compose up -d --build lb cleanup publisher worker +./scripts/wait-ready.sh diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..b77ec3f0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +exec python3 scripts/replicas.py stop diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..5a0ac7ac --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose stop --timeout 8 worker diff --git a/scripts/stop.sh b/scripts/stop.sh new file mode 100755 index 00000000..7dfffac9 --- /dev/null +++ b/scripts/stop.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose down --timeout 10 +echo "Контейнеры удалены; именованные volumes сохранены" diff --git a/scripts/wait-ready.sh b/scripts/wait-ready.sh new file mode 100755 index 00000000..da085770 --- /dev/null +++ b/scripts/wait-ready.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +exec python3 scripts/replicas.py wait diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..420cbd44 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,16 @@ +"""Общий запуск""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.app:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + access_log=False, + timeout_graceful_shutdown=5, + ) diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..76f31bb8 --- /dev/null +++ b/src/app.py @@ -0,0 +1,187 @@ +"""HTTP-адреса, определение пользователя и обработка запросов""" + +from contextlib import asynccontextmanager +import json +import logging +import re +from time import perf_counter +from uuid import UUID, uuid4 +from urllib.parse import quote + +from fastapi import FastAPI, Request, Response +from fastapi.responses import StreamingResponse +from starlette.concurrency import run_in_threadpool +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException + +from src import attachments, auth, db, migrate, object_store, passwords, storage +from src.errors import ApiError, error_response +from src.models import CreateChat, CreateMessage, CreateUser +from src.config import INSTANCE_ID + + +@asynccontextmanager +async def lifespan(app): + await run_in_threadpool(db.pool.open, wait=True, timeout=15) + try: + await run_in_threadpool(migrate.check) + yield + finally: + await run_in_threadpool(db.pool.close) + + +app = FastAPI(title="Учебный мессенджер", redirect_slashes=False, lifespan=lifespan) +logger = logging.getLogger("uvicorn.error") + + +@app.middleware("http") +async def handle_request(request: Request, call_next): + request.state.request_id = str(uuid4()) + started = perf_counter() + path = request.url.path + try: + is_registration = request.method == "POST" and path == "/api/v1/users" + is_login = request.method == "POST" and path == "/api/v1/auth/sessions" + is_upload = request.method == "POST" and re.fullmatch(r"/api/v1/chats/[^/]+/attachments", path) + if is_login: + request.state.user = await run_in_threadpool(auth.basic_user, request.headers.get("Authorization")) + if await request.body(): + raise ApiError(400, "invalid_request", "Вход через Basic не принимает тело запроса") + elif path.startswith("/api/v1/") and not is_registration: + request.state.user = await run_in_threadpool(auth.current_user, request.headers.get("Authorization")) + if request.method == "POST" and path.startswith("/api/v1/") and not is_login and not is_upload: + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + try: + json.loads((await request.body()).decode("utf-8")) + except (ValueError, RecursionError): + raise ApiError(400, "invalid_request", "Ожидается корректный JSON в UTF-8") + response = await call_next(request) + except ApiError as error: + response = error_response(request, error) + except Exception as error: + # Не печатаем тело запроса или текст исключения: там могут быть сообщения. + logger.error("%s request_id=%s", type(error).__name__, request.state.request_id) + response = error_response(request, ApiError(500, "internal_error", "Внутренняя ошибка сервера")) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + logger.info( + "%s %s status=%s duration_ms=%.2f request_id=%s", + request.method, path, response.status_code, + (perf_counter() - started) * 1000, request.state.request_id, + ) + return response + + +@app.exception_handler(ApiError) +def handle_api_error(request: Request, error: ApiError): + return error_response(request, error) + + +@app.exception_handler(RequestValidationError) +def handle_validation_error(request: Request, error: RequestValidationError): + # Контракт требует 400 и наш формат ошибки, а не стандартный ответ FastAPI. + return error_response(request, ApiError(400, "invalid_request", "Неверные поля или параметры запроса")) + + +@app.exception_handler(HTTPException) +def handle_http_error(request: Request, error: HTTPException): + code = "not_found" if error.status_code == 404 else "invalid_request" + message = "Адрес не найден" if error.status_code == 404 else "Недопустимый HTTP-запрос" + response = error_response(request, ApiError(error.status_code, code, message)) + if error.headers: + response.headers.update(error.headers) + return response + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + db.ready() + auth.ready() + object_store.ready() + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def create_user(data: CreateUser): + return storage.create_user(data.username, data.display_name, passwords.hash_password(data.password)) + + +@app.get("/api/v1/users/me") +def current_user(request: Request): + return request.state.user + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(request: Request, data: CreateChat): + return storage.create_chat(request.state.user["id"], data.title, data.member_ids) + + +@app.get("/api/v1/chats") +def list_chats(request: Request, limit: str = "50", cursor: str | None = None): + user_id = request.state.user["id"] + return storage.list_chats(user_id, limit, cursor) + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(request: Request, chat_id: UUID): + return storage.get_chat(request.state.user["id"], str(chat_id)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(request: Request, chat_id: UUID, data: CreateMessage): + keys = request.headers.getlist("Idempotency-Key") + if len(keys) != 1 or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", keys[0]) is None: + raise ApiError(400, "invalid_request", "Нужен Idempotency-Key: 1–128 символов ASCII [A-Za-z0-9._:-]") + return storage.create_message(request.state.user["id"], str(chat_id), data.text, keys[0], data.attachment_ids) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(request: Request, chat_id: UUID, filename: str): + return await attachments.upload(request, str(chat_id), filename) + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public_attachment(attachments.find(request.state.user["id"], str(attachment_id))) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(request: Request, attachment_id: UUID, variant: str = "original"): + row = attachments.find(request.state.user["id"], str(attachment_id)) + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение не готово") + info = row + key = row["object_key"] + if variant != "original": + info = next((item for item in row["variants"] if item["name"] == variant), None) + if info is None: + raise ApiError(404, "not_found", "Вариант вложения не найден") + key = row["object_key"].rsplit("/", 1)[0] + "/" + variant + response = object_store.get(key) + return StreamingResponse(attachments.chunks(response), media_type=info["content_type"], headers={ + "Content-Length": str(info["size_bytes"]), "X-Content-Type-Options": "nosniff", + "Content-Disposition": "attachment; filename*=UTF-8''" + quote(row["filename"], safe=""), + }) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(request: Request, chat_id: UUID, limit: str = "50", cursor: str | None = None): + return storage.list_messages(request.state.user["id"], str(chat_id), limit, cursor) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request): + return auth.create_session(request.state.user) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + auth.logout(request.headers.get("Authorization")) + return Response(status_code=204) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..b97ced9f --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,151 @@ +"""Приём и проверка -> durable S3 staging + PostgreSQL outbox -> 202.""" +import asyncio +import hashlib +import os +from tempfile import TemporaryFile +import unicodedata +from uuid import UUID, uuid4 +import warnings + +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from src import db, jobs, object_store, storage +from src.errors import ApiError + + +MAX_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +if not 1 <= MAX_BYTES <= 10485760 or not 1 <= MAX_PIXELS <= 40000000: + raise ValueError("Лимиты загрузки должны быть положительными и не превышать контракт") +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +warnings.filterwarnings("error", category=Image.DecompressionBombWarning) +# Максимум одно полное декодирование на реплику, независимо от числа клиентов. +upload_slot = asyncio.Semaphore(1) + + +def public_attachment(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": row["variants"], "error_code": row["error_code"], "created_at": storage.utc(row["created_at"]), + "updated_at": storage.utc(row["updated_at"])} + + +def validate_filename(filename): + if not 1 <= len(filename) <= 255 or not filename.strip() or any( + char in "/\\" or unicodedata.category(char) == "Cc" for char in filename + ): + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + try: + filename.encode("utf-8") + except UnicodeError: + raise ApiError(400, "invalid_request", "Недопустимое имя файла") from None + + +def validate_image(file, declared_type): + try: + file.seek(0) + with Image.open(file, formats=["PNG", "JPEG"]) as image: + true_type = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + if true_type != declared_type: + raise ApiError(415, "unsupported_media_type", "MIME не совпадает с содержимым") + image.verify() + # verify проверяет структуру/CRC, load дополнительно обнаруживает обрыв pixel data. + file.seek(0) + with Image.open(file, formats=["PNG", "JPEG"]) as image: + image.load() + return true_type + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG/JPEG или другой формат") from None + + +def lock_upload(conn, key, blocking=True): + function = "pg_advisory_xact_lock" if blocking else "pg_try_advisory_xact_lock" + return conn.execute(f"SELECT {function}(hashtextextended(%s, 4)) AS locked", (key,)).fetchone()["locked"] + + +def save(user_id, chat_id, filename, content_type, file, size, sha256): + attachment_id, job_id = uuid4(), uuid4() + staging_key = "attachments/staging/" + attachment_id.hex + final_key = "attachments/" + attachment_id.hex + "/v1/original" + # Журнал фиксируется до PUT: crash до записи job оставит управляемый orphan. + with db.connection() as conn: + storage.require_member(conn, user_id, chat_id) + conn.execute("INSERT INTO attachment_uploads(id, object_key) VALUES (%s, %s)", (attachment_id, staging_key)) + with db.connection() as conn: + conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'") + lock_upload(conn, staging_key) + if not conn.execute("SELECT 1 FROM attachment_uploads WHERE id=%s", (attachment_id,)).fetchone(): + raise ApiError(503, "unavailable", "Загрузка больше не активна") + object_store.put(staging_key, file, size, content_type) + row = conn.execute("""INSERT INTO attachments(id,chat_id,owner_id,object_key,filename, + content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *""", + (attachment_id, UUID(chat_id), UUID(user_id), final_key, filename, content_type, size, sha256)).fetchone() + conn.execute("""INSERT INTO attachment_jobs(id,attachment_id,staging_key,max_attempts) + VALUES (%s,%s,%s,%s)""", (job_id, attachment_id, staging_key, jobs.MAX_ATTEMPTS)) + conn.execute("INSERT INTO attachment_outbox(job_id,attempt) VALUES (%s,1)", (job_id,)) + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,)) + result = public_attachment(row) + # Снимок queued даже если worker успеет закончить до отправки ответа. + return result + + +async def upload(request, chat_id, filename): + user_id = request.state.user["id"] + await run_in_threadpool(storage.get_chat, user_id, chat_id) + validate_filename(filename) + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Ожидаются сырые bytes PNG/JPEG") + length = request.headers.get("Content-Length") + if length is not None: + if not length.isascii() or not length.isdecimal(): + raise ApiError(400, "invalid_request", "Неверный Content-Length") + if len(length) > 10 or int(length) > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + try: + await asyncio.wait_for(upload_slot.acquire(), timeout=20) + except TimeoutError: + raise ApiError(503, "unavailable", "Загрузки заняты; повторите позднее") from None + try: + with TemporaryFile() as file: + size = 0 + digest = hashlib.sha256() + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + except (TimeoutError, ClientDisconnect): + raise ApiError(400, "invalid_request", "Загрузка прервана или истёк таймаут") from None + true_type = await run_in_threadpool(validate_image, file, content_type) + return await run_in_threadpool(save, user_id, chat_id, filename, true_type, file, size, digest.hexdigest()) + finally: + upload_slot.release() + + +def find(user_id, attachment_id): + with db.connection() as conn: + row = conn.execute("""SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id + WHERE a.id=%s AND m.user_id=%s""", (UUID(attachment_id), UUID(user_id))).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +def chunks(response): + try: + while chunk := response.read(65536): + yield chunk + finally: + response.close() + response.release_conn() diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..a8db643d --- /dev/null +++ b/src/auth.py @@ -0,0 +1,95 @@ +"""Basic только для входа; Bearer-сессии с TTL и AOF находятся в Valkey.""" +import base64 +import binascii +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +import hashlib +import json +import secrets + +import redis + +from src import passwords, storage +from src.config import SESSION_TTL, secret +from src.errors import ApiError + + +sessions = redis.Redis.from_url( + secret("CACHE_URL"), decode_responses=True, + socket_connect_timeout=0.5, socket_timeout=0.5, max_connections=10, + retry=redis.retry.Retry(redis.backoff.NoBackoff(), 0), +) + + +@contextmanager +def available(): + try: + yield + except redis.RedisError: + raise ApiError(503, "unavailable", "Valkey временно недоступен") from None + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, "unauthorized", "Неверная или отсутствующая идентификация", {"WWW-Authenticate": challenge}) + + +def credential(header, scheme): + parts = (header or "").split() + if len(parts) != 2 or parts[0].lower() != scheme.lower(): + raise unauthorized(scheme == "Basic") + return parts[1] + + +def basic_user(header): + value = credential(header, "Basic") + try: + decoded = base64.b64decode(value, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise unauthorized(True) from None + user = storage.find_login(username) + if user is None or not passwords.matches(user["password_hash"], password): + raise unauthorized(True) + return storage.public_user(user) + + +def session_key(token): + return "session:" + hashlib.sha256(token.encode()).hexdigest() + + +def create_session(user): + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL) + value = {"user_id": user["id"], "expires_at": storage.utc(expires)} + with available(): + sessions.set(session_key(token), json.dumps(value), ex=SESSION_TTL) + return {"token": token, "token_type": "Bearer", "expires_at": value["expires_at"], "user": user} + + +def current_user(header): + token = credential(header, "Bearer") + with available(): + value = sessions.get(session_key(token)) + if value is None: + raise unauthorized() + data = json.loads(value) + if datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00")) <= datetime.now(timezone.utc): + raise unauthorized() + user = storage.find_user(data["user_id"]) + if user is None: + raise unauthorized() + return user + + +def logout(header): + token = credential(header, "Bearer") + with available(): + removed = sessions.delete(session_key(token)) + if not removed: + raise unauthorized() + + +def ready(): + with available(): + sessions.ping() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 00000000..254c6fe0 --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,38 @@ +"""Идемпотентное создание приватного bucket отдельной административной учёткой.""" +import os +import time +from urllib.parse import urlsplit + +from minio import Minio +from minio.error import MinioException, S3Error +import urllib3 + +from src.config import secret + + +def bootstrap(): + endpoint = urlsplit(os.environ.get("S3_ENDPOINT", "http://s3:9000")) + client = Minio(endpoint.netloc, access_key=secret("S3_ROOT_USER"), secret_key=secret("S3_ROOT_PASSWORD"), + secure=endpoint.scheme == "https", region=os.environ.get("S3_REGION", "us-east-1"), + http_client=urllib3.PoolManager(timeout=urllib3.Timeout(connect=1, read=3), retries=False)) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + deadline = time.monotonic() + 60 + while True: + try: + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + print("Private bucket is ready; static IAM permits only authenticated accounts", flush=True) + return + except S3Error as error: + if error.code in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + return + if time.monotonic() >= deadline: + raise RuntimeError("S3 bootstrap failed") from None + except (MinioException, urllib3.exceptions.HTTPError, OSError): + if time.monotonic() >= deadline: + raise RuntimeError("S3 bootstrap timed out") from None + time.sleep(1) + + +if __name__ == "__main__": + bootstrap() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..b918c146 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,38 @@ +"""Один durable Stream, consumer group; ACK только после эффекта/устойчивого retry.""" +import json + +import redis + +from src.config import secret + +STREAM = 'attachments:v1' +GROUP = 'image-workers' + + +def connect(): + return redis.Redis.from_url(secret('BROKER_URL'), decode_responses=True, + socket_connect_timeout=2, socket_timeout=3, + retry_on_timeout=False) + + +def bootstrap(client): + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): + raise + + +def publish(client, job): + payload = {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], + 'attempt': job['attempt'], 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z')} + return client.xadd(STREAM, {'job': json.dumps(payload)}) + + +def acknowledge(client, message_id): + # Единственная группа: подтверждённые записи можно удалить. Pending не trim-им. + with client.pipeline(transaction=True) as pipe: + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + pipe.execute() diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 00000000..0a698080 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,87 @@ +"""Очистка orphan PUT и staging только ready/просроченных failed; текущую работу не трогает.""" +import argparse +import json +import logging +import os +import time + +from src import db, object_store +from src.attachments import lock_upload + + +GRACE_SECONDS = int(os.environ.get("UPLOAD_CLEANUP_GRACE_SECONDS", "3600")) +if GRACE_SECONDS < 60: + raise ValueError("Безопасный grace period должен быть не меньше 60 секунд") + + +def cleanup_orphans(): + result = {"removed": 0, "kept_ready": 0, "skipped_active": 0} + with db.connection() as conn: + candidates = conn.execute("""SELECT id, object_key FROM attachment_uploads + WHERE created_at < clock_timestamp() - %s * interval '1 second' + ORDER BY created_at LIMIT 100""", (GRACE_SECONDS,)).fetchall() + for row in candidates: + with db.connection() as conn: + if not lock_upload(conn, row["object_key"], blocking=False): + result["skipped_active"] += 1 + continue + pending = conn.execute("""SELECT 1 FROM attachment_uploads WHERE id=%s + AND created_at < clock_timestamp() - %s * interval '1 second' FOR UPDATE""", + (row["id"], GRACE_SECONDS)).fetchone() + if not pending: + continue + ready = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (row["object_key"],)).fetchone() + if ready: + result["kept_ready"] += 1 + else: + object_store.remove(row["object_key"]) + result["removed"] += 1 + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (row["id"],)) + return result + + +RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) +if RETENTION < 60: + raise ValueError("STAGING_RETENTION_SECONDS должен быть >=60") + + +def cleanup_staging(): + removed = 0 + with db.connection() as conn: + rows = conn.execute("""SELECT j.*,a.status,a.object_key FROM attachment_jobs j + JOIN attachments a ON a.id=j.attachment_id WHERE j.staging_deleted_at IS NULL + AND (a.status='ready' OR (a.status='failed' AND a.updated_at Image.MAX_IMAGE_PIXELS: + raise ValueError('pixel_limit') + image.load() + ImageOps.exif_transpose(image, in_place=True) + if image.mode in ('RGBA', 'LA') or 'transparency' in image.info: + rgba = image.convert('RGBA') + rgb = Image.new('RGB', image.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + rgba.close() + else: + rgb = image.convert('RGB') + rgb.info.clear() + variants = [] + for name, size, encoding, mime in [ + ('thumbnail', fit_size(rgb.size, 256), 'JPEG', 'image/jpeg'), + ('crop', (128, 128), 'JPEG', 'image/jpeg'), + ('optimized', fit_size(rgb.size, 1280), 'WEBP', 'image/webp'), + ]: + variant = ImageOps.fit(rgb, size, method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) if name == 'crop' else rgb.resize(size, Image.Resampling.LANCZOS) + variant.info.clear() + buffer = io.BytesIO() + variant.save(buffer, format=encoding, quality=85 if encoding == 'JPEG' else 80, + **({'method': 4} if encoding == 'WEBP' else {})) + data = buffer.getvalue() + (output / name).write_bytes(data) + variants.append({'name': name, 'content_type': mime, 'size_bytes': len(data), + 'width': size[0], 'height': size[1], 'sha256': hashlib.sha256(data).hexdigest()}) + variant.close() + rgb.close() + (output / 'variants.json').write_text(json.dumps(variants)) + + +if __name__ == '__main__': + memory = int(os.environ.get('WORKER_TASK_MEMORY_MB', '640')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + seconds = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60')) + resource.setrlimit(resource.RLIMIT_CPU, (seconds, seconds + 1)) + try: + from src import faults + faults.hit(sys.argv[3], 'transform') + transform(Path(sys.argv[1]), Path(sys.argv[2])) + except (ValueError, OSError, SyntaxError, Image.DecompressionBombWarning, Image.DecompressionBombError): + sys.exit(65) # permanent invalid_image; stderr не содержит внутренние данные. + except MemoryError: + sys.exit(66) diff --git a/src/job_admin.py b/src/job_admin.py new file mode 100644 index 00000000..842301b2 --- /dev/null +++ b/src/job_admin.py @@ -0,0 +1,29 @@ +"""Локальная операторская CLI: failed registry и безопасный replay по job_id.""" +import argparse +import json + +from src import db, jobs + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + sub.add_parser('failed') + replay = sub.add_parser('replay') + replay.add_argument('job_id') + args = parser.parse_args() + db.pool.open(wait=True, timeout=15) + try: + if args.command == 'replay': + jobs.replay(args.job_id) + print('Replay queued with audit; job_id=' + args.job_id) + else: + with db.connection() as conn: + rows = conn.execute("""SELECT j.id AS job_id,j.attachment_id,j.attempt,j.max_attempts, + a.error_code,j.last_error,j.staging_deleted_at FROM attachment_jobs j + JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY a.updated_at""").fetchall() + print(json.dumps(rows, default=str, indent=2)) + except ValueError as error: + parser.exit(2, str(error) + '\n') + finally: + db.pool.close() diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..360b7312 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,95 @@ +"""Транзакционные переходы состояний, bounded retry и явный audit replay.""" +import os +import random +from uuid import UUID, uuid4 + +from src import db + +MAX_ATTEMPTS = int(os.environ.get('JOB_MAX_ATTEMPTS', '5')) +TIMEOUT = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60')) +LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '90')) +BACKOFF = float(os.environ.get('JOB_BACKOFF_SECONDS', '2')) +if not 1 <= MAX_ATTEMPTS <= 20 or TIMEOUT < 1 or LEASE < TIMEOUT + 5 or BACKOFF <= 0: + raise ValueError('Неверные retry/timeout/lease настройки') + + +def event(conn, job, name, error=None): + conn.execute('INSERT INTO attachment_job_events(job_id,attempt,event,error_code) VALUES (%s,%s,%s,%s)', + (job['id'], job['attempt'], name, error)) + + +def locked(conn, job_id): + return conn.execute('''SELECT j.*, a.status, a.object_key, a.variants, a.sha256, a.size_bytes, + a.content_type FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE j.id=%s FOR UPDATE OF j,a''', (job_id,)).fetchone() + + +def claim(payload): + with db.connection() as conn: + job = locked(conn, UUID(payload['job_id'])) + if not job or str(job['attachment_id']) != payload['attachment_id'] or job['pipeline_version'] != payload['pipeline_version']: + return 'obsolete', None + if job['status'] in {'ready', 'failed'}: + return job['status'], job + if payload['attempt'] != job['attempt']: + return 'obsolete', None + if job['status'] != 'queued': + return 'busy', None + row = conn.execute('''UPDATE attachment_jobs SET lease_token=%s, + lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s + AND next_attempt_at<=clock_timestamp() RETURNING *''', (uuid4(), LEASE, job['id'])).fetchone() + if not row: + return 'busy', None + job.update(row) + conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],)) + event(conn, job, 'processing') + return 'claimed', job + + +def retry_or_fail(conn, job, code, permanent=False): + # replay_count задаёт новый ограниченный бюджет, сохраняя глобальный номер attempt. + exhausted = job['attempt'] >= job['max_attempts'] + if permanent or exhausted: + final_code = code if permanent else 'retry_exhausted' + conn.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + (final_code, job['attachment_id'])) + conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=%s WHERE id=%s', (code, job['id'])) + event(conn, job, 'failed', final_code) + return + delay = random.uniform(0.5, 1.5) * min(60, BACKOFF * 2 ** min(job['attempt'] - 1, 10)) + conn.execute('''UPDATE attachment_jobs SET attempt=attempt+1, lease_token=NULL,lease_until=NULL, + last_error=%s,next_attempt_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s''', (code, delay, job['id'])) + conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],)) + conn.execute('''INSERT INTO attachment_outbox(job_id,attempt,available_at) + SELECT id,attempt,next_attempt_at FROM attachment_jobs WHERE id=%s''', (job['id'],)) + event(conn, job, 'retry', code) + + +def fail(job, code, permanent=False): + with db.connection() as conn: + current = locked(conn, job['id']) + if current['status'] == 'processing' and current['lease_token'] == job['lease_token']: + retry_or_fail(conn, current, code, permanent) + + +def recover(): + with db.connection() as conn: + rows = conn.execute('''SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE a.status='processing' AND j.lease_until size: + last = items[size - 1] + value = {"scope": scope, "created_at": last["created_at"], "id": last["id"]} + cursor = base64.urlsafe_b64encode(json.dumps(value).encode()).decode() + return {"items": items[:size], "next_cursor": cursor} diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 00000000..bc980660 --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,22 @@ +"""Argon2id. Одновременно считаются не более двух хешей.""" +from threading import Semaphore + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError + + +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +slots = Semaphore(2) + + +def hash_password(password): + with slots: + return hasher.hash(password) + + +def matches(encoded, password): + with slots: + try: + return hasher.verify(encoded, password) + except VerificationError: + return False diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 00000000..7fd12b5d --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,39 @@ +"""Outbox dispatcher: повтор XADD при неопределённом исходе безопасен.""" +import logging +import time + +from src import broker, db, faults, jobs + + +def publish_one(client): + with db.connection() as conn: + job = conn.execute('''SELECT j.*,o.attempt AS delivery_attempt FROM attachment_outbox o + JOIN attachment_jobs j ON j.id=o.job_id WHERE o.published_at IS NULL + AND o.available_at<=clock_timestamp() ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED''').fetchone() + if not job: + return False + job['attempt'] = job['delivery_attempt'] + faults.hit(job['id'], 'before_publish') + broker.publish(client, job) + faults.hit(job['id'], 'after_publish') + conn.execute('UPDATE attachment_outbox SET published_at=clock_timestamp() WHERE job_id=%s AND attempt=%s', + (job['id'], job['attempt'])) + return True + + +if __name__ == '__main__': + db.pool.open(wait=True, timeout=30) + client = broker.connect() + try: + while True: + try: + jobs.recover() + broker.bootstrap(client) + if not publish_one(client): + time.sleep(0.25) + except Exception as error: + logging.error('Outbox postponed: %s', type(error).__name__) + time.sleep(1) + finally: + client.close() + db.pool.close() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..4bf40f7b --- /dev/null +++ b/src/storage.py @@ -0,0 +1,156 @@ +"""Параметризованный SQL. Одна функция записи — одна транзакция.""" +from uuid import UUID, uuid4 + +from psycopg.errors import ForeignKeyViolation, UniqueViolation + +from src import db, pagination +from src.errors import ApiError +from src.config import IDEMPOTENCY_TTL + + +def utc(value): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def public_user(row): + return {"id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": utc(row["created_at"])} + + +def public_chat(row): + return {"id": str(row["id"]), "title": row["title"], + "member_ids": [str(value) for value in row["member_ids"]], "created_at": utc(row["created_at"])} + + +def public_message(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": utc(row["created_at"]), + "attachment_ids": [str(value) for value in row.get("attachment_ids", [])]} + + +MESSAGE_ATTACHMENTS = """ARRAY(SELECT attachment_id FROM message_attachments + WHERE message_id=m.id ORDER BY position) AS attachment_ids""" + + +def create_user(username, display_name, password_hash): + try: + with db.connection() as conn: + row = conn.execute("""INSERT INTO users(id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING *""", (uuid4(), username, display_name, password_hash)).fetchone() + result = public_user(row) + return result + except UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя уже занято") from None + + +def find_login(username): + with db.connection() as conn: + return conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + + +def find_user(user_id): + with db.connection() as conn: + row = conn.execute("SELECT * FROM users WHERE id=%s", (UUID(user_id),)).fetchone() + return public_user(row) if row else None + + +def create_chat(user_id, title, member_ids): + if user_id not in member_ids: + raise ApiError(400, "invalid_request", "Создатель должен входить в member_ids") + try: + with db.connection() as conn: + row = conn.execute("INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING *", (uuid4(), title)).fetchone() + with conn.cursor() as cur: + cur.executemany("INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)", + [(row["id"], UUID(member)) for member in member_ids]) + row["member_ids"] = member_ids + result = public_chat(row) + return result + except ForeignKeyViolation: + raise ApiError(400, "invalid_request", "Все участники должны существовать") from None + + +def require_member(conn, user_id, chat_id): + row = conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (UUID(chat_id), UUID(user_id))).fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + + +def get_chat(user_id, chat_id): + with db.connection() as conn: + require_member(conn, user_id, chat_id) + row = conn.execute("""SELECT c.*, ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids + FROM chats c WHERE c.id=%s""", (UUID(chat_id),)).fetchone() + return public_chat(row) + + +def list_chats(user_id, limit, cursor): + scope = "chats:" + user_id + size, anchor = pagination.parameters(limit, cursor, scope) + with db.connection() as conn: + if anchor: + found = conn.execute("""SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id + WHERE m.user_id=%s AND (c.created_at, c.id)=(%s, %s)""", (UUID(user_id), *anchor)).fetchone() + if not found: + raise ApiError(400, "invalid_request", "Неверный cursor") + condition = "AND (c.created_at, c.id) > (%s, %s)" if anchor else "" + params = (UUID(user_id), *(anchor or ()), size + 1) + rows = conn.execute("""WITH selected AS ( + SELECT c.* FROM chats c JOIN chat_members own ON own.chat_id=c.id + WHERE own.user_id=%s """ + condition + """ ORDER BY c.created_at, c.id LIMIT %s + ) SELECT c.*, ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids + FROM selected c ORDER BY c.created_at, c.id""", params).fetchall() + return pagination.page([public_chat(row) for row in rows], size, scope) + + +def create_message(user_id, chat_id, text, key, attachment_ids=()): + attachment_ids = [UUID(value) for value in attachment_ids] + with db.connection() as conn: + require_member(conn, user_id, chat_id) + # Одна блокировка на область ключа во всех API-процессах. + # Побочный эффект и результат повтора фиксируются одной транзакцией. + scope = user_id + ":POST:/api/v1/chats/" + chat_id + "/messages:" + key + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,)) + previous = conn.execute("SELECT m.*, i.request_text, i.request_attachment_ids, " + MESSAGE_ATTACHMENTS + """ FROM message_idempotency i + JOIN messages m ON m.id=i.message_id + WHERE i.user_id=%s AND i.chat_id=%s AND i.key=%s AND i.expires_at>clock_timestamp()""", + (UUID(user_id), UUID(chat_id), key)).fetchone() + if previous: + if previous["request_text"] != text or previous["request_attachment_ids"] != attachment_ids: + raise ApiError(409, "conflict", "Этот ключ уже использован с другим сообщением") + return public_message(previous) + if attachment_ids: + available = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)", + (UUID(chat_id), attachment_ids)).fetchall() + if len(available) != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено в этом чате") + if any(item["status"] == "failed" for item in available): + raise ApiError(409, "conflict", "Вложение не обработано") + conn.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", + (UUID(user_id), UUID(chat_id), key)) + row = conn.execute("""INSERT INTO messages(id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s) + RETURNING *""", (uuid4(), UUID(chat_id), UUID(user_id), text)).fetchone() + for position, attachment_id in enumerate(attachment_ids): + conn.execute("INSERT INTO message_attachments(message_id, attachment_id, position) VALUES (%s, %s, %s)", + (row["id"], attachment_id, position)) + conn.execute("""INSERT INTO message_idempotency(user_id, chat_id, key, request_text, request_attachment_ids, message_id, expires_at) + VALUES (%s, %s, %s, %s, %s, %s, clock_timestamp() + %s * interval '1 second')""", + (UUID(user_id), UUID(chat_id), key, text, attachment_ids, row["id"], IDEMPOTENCY_TTL)) + row["attachment_ids"] = attachment_ids + result = public_message(row) + return result + + +def list_messages(user_id, chat_id, limit, cursor): + scope = "messages:" + chat_id + size, anchor = pagination.parameters(limit, cursor, scope) + with db.connection() as conn: + require_member(conn, user_id, chat_id) + if anchor: + found = conn.execute("SELECT 1 FROM messages WHERE chat_id=%s AND (created_at, id)=(%s, %s)", (UUID(chat_id), *anchor)).fetchone() + if not found: + raise ApiError(400, "invalid_request", "Неверный cursor") + condition = "AND (m.created_at, m.id) > (%s, %s)" if anchor else "" + rows = conn.execute("SELECT m.*, " + MESSAGE_ATTACHMENTS + " FROM messages m WHERE m.chat_id=%s " + condition + + " ORDER BY m.created_at, m.id LIMIT %s", (UUID(chat_id), *(anchor or ()), size + 1)).fetchall() + return pagination.page([public_message(row) for row in rows], size, scope) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..5a9b66ce --- /dev/null +++ b/src/worker.py @@ -0,0 +1,191 @@ +"""Worker supervisors: concurrency процессов, prefetch=1, дочерний resize с timeout.""" +from datetime import datetime +import hashlib +import json +import logging +import multiprocessing +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +from uuid import UUID, uuid4 + +from psycopg.types.json import Jsonb + +from src import broker, db, faults, jobs, object_store +from src.errors import ApiError + + +class ProcessingError(Exception): + def __init__(self, code, permanent=False): + self.code, self.permanent = code, permanent + + +def verify_ready(job): + # Старые ready из lab4 не имеют job; у новой job всегда все три варианта. + if {v['name'] for v in job['variants']} != {'thumbnail', 'crop', 'optimized'}: + raise ProcessingError('incomplete_result') + for key, size in [(job['object_key'], job['size_bytes'])] + [ + (job['object_key'].rsplit('/', 1)[0] + '/' + v['name'], v['size_bytes']) for v in job['variants'] + ]: + with object_store.available(): + if object_store.client.stat_object(object_store.BUCKET, key).size != size: + raise ProcessingError('incomplete_result') + + +def process(job): + with TemporaryDirectory(prefix='image-job-') as directory: + output = Path(directory) + source = output / 'original' + response = object_store.get(job['staging_key']) + size, digest = 0, hashlib.sha256() + try: + with source.open('wb') as file: + while chunk := response.read(65536): + size += len(chunk) + if size > job['size_bytes'] or size > 10485760: + raise ProcessingError('invalid_image', True) + digest.update(chunk) + file.write(chunk) + finally: + response.close() + response.release_conn() + if size != job['size_bytes'] or digest.hexdigest() != job['sha256']: + raise ProcessingError('invalid_image', True) + try: + result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', str(source), str(output), str(job['id'])], + timeout=jobs.TIMEOUT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + raise ProcessingError('processing_timeout') from None + if result.returncode: + raise ProcessingError('invalid_image' if result.returncode == 65 else 'resource_limit', result.returncode == 65) + variants = json.loads((output / 'variants.json').read_text()) + # Короткая транзакция fence перед side effects и публикацией результата. + with db.connection() as conn: + conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'") + current = jobs.locked(conn, job['id']) + active = conn.execute('SELECT lease_until>clock_timestamp() AS active FROM attachment_jobs WHERE id=%s', (job['id'],)).fetchone()['active'] + if current['status'] != 'processing' or current['lease_token'] != job['lease_token'] or not active: + return False + with source.open('rb') as file: + object_store.put(job['object_key'], file, job['size_bytes'], job['content_type']) + prefix = job['object_key'].rsplit('/', 1)[0] + '/' + for variant in variants: + with (output / variant['name']).open('rb') as file: + object_store.put(prefix + variant['name'], file, variant['size_bytes'], variant['content_type']) + faults.hit(job['id'], 'after_objects') + conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (Jsonb(variants), job['attachment_id'])) + conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=NULL WHERE id=%s', (job['id'],)) + jobs.event(conn, job, 'ready') + faults.hit(job['id'], 'before_ack') + return True + + +def handle(client, message_id, fields): + try: + payload = json.loads(fields['job']) + required = {'schema_version', 'type', 'job_id', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(payload, dict) or not required <= payload.keys() or payload.keys() - required - {'traceparent', 'tracestate'}: + raise ValueError('invalid job') + if type(payload['schema_version']) is not int or payload['schema_version'] != 1 or payload['type'] != 'attachment.process.v1': + raise ValueError('invalid job') + payload['job_id'] = str(UUID(payload['job_id'])) + payload['attachment_id'] = str(UUID(payload['attachment_id'])) + if type(payload['attempt']) is not int or payload['attempt'] < 1 or type(payload['pipeline_version']) is not int or payload['pipeline_version'] != 1: + raise ValueError('invalid job') + if not isinstance(payload['created_at'], str) or datetime.fromisoformat(payload['created_at'].replace('Z', '+00:00')).tzinfo is None: + raise ValueError('invalid job') + if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None): + raise ValueError('invalid trace context') + if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512): + raise ValueError('invalid trace context') + except (ValueError, TypeError, KeyError): + logging.error('Invalid broker message rejected') + broker.acknowledge(client, message_id) + return + state, job = jobs.claim(payload) + if state == 'busy': + return + if state == 'ready': + verify_ready(job) + if state == 'claimed': + try: + faults.hit(job['id'], 'before_process') + if not process(job): + return + except ProcessingError as error: + jobs.fail(job, error.code, error.permanent) + except Exception: + jobs.fail(job, 'storage_unavailable') + # ready/failed/obsolete или записанный retry; при потере COMMIT исключение не даёт ACK. + broker.acknowledge(client, message_id) + + +def consume(): + db.pool.open(wait=True, timeout=30) + client = broker.connect() + consumer = 'worker-' + uuid4().hex + cursor = '0-0' + try: + while True: + try: + broker.bootstrap(client) + claimed = client.xautoclaim(broker.STREAM, broker.GROUP, consumer, + min_idle_time=jobs.LEASE * 1000, start_id=cursor, count=1) + cursor = claimed[0] + entries = claimed[1] + if not entries: + batches = client.xreadgroup(broker.GROUP, consumer, {broker.STREAM: '>'}, count=1, block=1000) + entries = batches[0][1] if batches else [] + for message_id, fields in entries: + handle(client, message_id, fields) + except Exception as error: + logging.error('Worker postponed: %s', type(error).__name__) + time.sleep(1) + finally: + client.close() + db.pool.close() + + +def main(): + concurrency = int(os.environ.get('WORKER_CONCURRENCY', '2')) + if not 1 <= concurrency <= 2: + raise ValueError('WORKER_CONCURRENCY: 1 или 2 для заданного memory limit') + context = multiprocessing.get_context('spawn') + processes = [] + stopping = False + def stop(signum, frame): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + try: + while not stopping: + alive = [] + for child in processes: + if child.is_alive(): + alive.append(child) + else: + child.join() + processes = alive + while len(processes) < concurrency: + child = context.Process(target=consume) + child.start() + processes.append(child) + time.sleep(0.25) + finally: + for child in processes: + child.terminate() + for child in processes: + child.join(timeout=5) + if child.is_alive(): + child.kill() + child.join() + + +if __name__ == '__main__': + main() diff --git a/student-tests/attachment_checks.py b/student-tests/attachment_checks.py new file mode 100644 index 00000000..63562be1 --- /dev/null +++ b/student-tests/attachment_checks.py @@ -0,0 +1,275 @@ +"""Реальные отказы/очистка/память уровня 4; только временный тестовый стенд.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import io +import json +import os +import socket +import subprocess +import threading +import time +from urllib.parse import urlsplit +from uuid import uuid4 + +from PIL import Image + +from helpers import Client, ROOT +from client import Failure + + +def run_checks(base, env, compose, apis, script, sql): + result = {} + client = Client(base, timeout=30) + alice, bob, _, chat = client.bootstrap() + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + upload_path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png" + + def upload(): + return client.call("POST", upload_path, 201, actor=alice, raw=png, + headers={"Content-Type": "image/png"}, schema="Attachment") + + def inside(code, data=None): + completed = subprocess.run(["docker", "exec", "-i", apis()[0]["Id"], "python", "-c", code], + input=json.dumps(data), capture_output=True, text=True, timeout=45, check=True) + return json.loads(completed.stdout) + + def cleanup(): + return inside("from src import db; from src.cleanup import cleanup_once; import json; " + "db.pool.open(wait=True); print(json.dumps(cleanup_once())); db.pool.close()") + + def stat(key): + return inside("import json,sys; from src import object_store; from minio.error import S3Error\n" + "key=json.load(sys.stdin)\n" + "try:\n r=object_store.client.stat_object(object_store.BUCKET,key); print(json.dumps({'size':r.size}))\n" + "except S3Error as e:\n print(json.dumps({'error':e.code}))", key) + + def age(key): + sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours' WHERE object_key='" + key + "'") + + compose("stop", "cleanup") + try: + for _ in range(2): + compose("run", "--rm", "s3-bootstrap") + ready = upload()["json"] + key = sql("SELECT object_key FROM attachments WHERE id='" + ready["id"] + "'") + backend_id = compose("ps", "-q", "s3-storage") + backend = json.loads(subprocess.check_output(["docker", "inspect", backend_id], text=True))[0] + assert not set(backend["NetworkSettings"]["Networks"]) & set(apis()[0]["NetworkSettings"]["Networks"]) + backend_ip = next(iter(backend["NetworkSettings"]["Networks"].values()))["IPAddress"] + # Вызов S3 напрямую из контейнера клиента, минуя HTTP API приложения. + result["private_bucket"] = inside(""" +import io,json,socket,sys +from urllib.error import HTTPError +from urllib.request import urlopen +from minio.error import S3Error +from src import object_store +data=json.load(sys.stdin); key=data['key'] +try: + urlopen('http://s3:9000/'+object_store.BUCKET+'/'+key,timeout=3) + raise AssertionError('public original') +except HTTPError as e: + assert e.code == 403 +for operation in (lambda: object_store.client.make_bucket('forbidden-bucket'), + lambda: object_store.client.put_object(object_store.BUCKET,'outside-prefix',io.BytesIO(b'x'),1)): + try: + operation() + raise AssertionError('excess application permissions') + except S3Error as e: + assert e.code == 'AccessDenied', e.code +for port in (9333,8888,8080): + with socket.socket() as connection: + connection.settimeout(1) + assert connection.connect_ex(('s3',port)) != 0, 'admin/backend port open' + with socket.socket() as connection: + connection.settimeout(.5) + assert connection.connect_ex((data['backend_ip'],port)) != 0, 'storage network reachable from API' +print(json.dumps({'anonymous_get':403,'create_bucket':'AccessDenied','write_outside_prefix':'AccessDenied', + 'admin_ports_unreachable':[9333,8888,8080],'storage_network_isolated':True,'bootstrap_runs':2})) +""", {"key":key, "backend_ip":backend_ip}) + print("PASS: private S3, limited credentials, isolated administration, repeated bootstrap", flush=True) + + # Останавливаем именно реплику, которая ответила на upload, а затем читаем другой. + uploaded = upload() + target = uploaded["headers"]["X-Instance-Id"] + stopped_at = time.monotonic() + subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=15) + script("wait-ready.sh") + while True: + try: + response = client.call("GET", "/api/v1/attachments/" + uploaded["json"]["id"] + "/content", actor=bob) + break + except Failure: + assert time.monotonic() - stopped_at < 10, "download failover exceeded 10 seconds" + time.sleep(.1) + assert response["bytes"] == png and response["headers"]["X-Instance-Id"] != target + stopped = next(info for info in apis() if info["Config"]["Hostname"] == target) + assert not stopped["State"]["Running"] + result["upload_replica_failover"] = {"uploaded_by": target, "stopped_container": stopped["Id"], + "downloaded_by": response["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(png).hexdigest(), + "recovery_seconds":round(time.monotonic()-stopped_at,3)} + compose("start", "api") + script("wait-ready.sh") + print("PASS: original downloaded through another replica after stopping uploader", flush=True) + + # Незавершённое HTTP-тело не достигает S3 и не создаёт метаданных/журнала. + before = sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") + address = urlsplit(base) + with socket.create_connection((address.hostname, address.port), timeout=3) as connection: + headers = ("POST " + upload_path + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer " + alice["token"] + + "\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n").encode() + connection.sendall(headers + png[:30]) + time.sleep(0.5) + assert sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") == before + result["interrupted_body"] = {"metadata_or_journal_created": False} + + # Детерминированный реальный отказ PostgreSQL ПОСЛЕ успешного S3 PUT. + sql("""CREATE FUNCTION lab4_reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected storage failure' USING ERRCODE='08006'; END $$; + CREATE TRIGGER lab4_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_reject_attachment();""") + try: + failure = client.call("POST", upload_path, 503, actor=alice, raw=png, headers={"Content-Type": "image/png"}) + assert failure["json"]["error"]["code"] == "unavailable" + finally: + sql("DROP TRIGGER lab4_reject ON attachments; DROP FUNCTION lab4_reject_attachment()") + pending_key = sql("SELECT object_key FROM attachment_uploads ORDER BY created_at DESC LIMIT 1") + assert stat(pending_key)["size"] == len(png) + assert sql("SELECT count(*) FROM attachments WHERE object_key='" + pending_key + "'") == "0" + assert cleanup()["removed"] == 0 + age(pending_key) + lock_code = ("from src import db; from src.attachments import lock_upload; import sys\n" + "db.pool.open(wait=True)\nwith db.connection() as c:\n" + " lock_upload(c,'" + pending_key + "'); print('locked',flush=True); sys.stdin.readline()\n" + "db.pool.close()") + holder = subprocess.Popen(["docker", "exec", "-i", apis()[0]["Id"], "python", "-u", "-c", lock_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert holder.stdout.readline().strip() == "locked" + assert cleanup()["skipped_active"] == 1 + assert stat(pending_key)["size"] == len(png) + finally: + holder.communicate("\n", timeout=10) + assert cleanup()["removed"] == 1 + assert stat(pending_key).get("error") == "NoSuchKey" + assert cleanup()["removed"] == 0 + # Даже устаревшая ошибочная запись журнала не позволяет удалить ready-файл. + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','" + key + + "',clock_timestamp()-interval '2 hours')") + assert cleanup()["kept_ready"] == 1 + assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png + result["compensation"] = {"failure_after_put":503, "orphan_recorded":True, "fresh_kept":True, + "active_locked_kept":True, "expired_removed":True, "ready_kept":True, "repeat_safe":True} + print("PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads", flush=True) + + try: + compose("stop", "--timeout", "8", "s3") + result["s3_outage"] = inside(""" +import json,sys +from urllib.request import Request,urlopen +from urllib.error import HTTPError +data=json.load(sys.stdin) +statuses={} +for name,method,path,body in [('live','GET','/health/live',None),('ready','GET','/health/ready',None), + ('download','GET','/api/v1/attachments/'+data['id']+'/content',None), + ('upload','POST',data['upload'],bytes.fromhex(data['png']))]: + req=Request('http://127.0.0.1:8080'+path,data=body,method=method, + headers={'Authorization':'Bearer '+data['token'],'Content-Type':'image/png'}) + try: response=urlopen(req,timeout=8) + except HTTPError as error: response=error + with response: + statuses[name]=response.status + payload=json.load(response) + if name!='live': + assert response.status==503 and payload['error']['code']=='unavailable' + assert payload['error']['request_id']==response.headers['X-Request-Id'] +assert statuses['live']==200 +print(json.dumps(statuses)) +""", {"id":ready["id"], "token":alice["token"], "upload":upload_path, "png":png.hex()}) + finally: + compose("start", "s3") + script("wait-ready.sh") + assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png + sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours'") + assert cleanup()["removed"] >= 1 + result["s3_outage"]["original_preserved_after_recovery"] = True + print("PASS: S3 outage gives API 503; recovery preserves original and session", flush=True) + + # Доказываем работу постоянного worker, а не только ручного вызова cleanup_once. + inside("import io,json; from src import object_store; " + "object_store.put('attachments/cleanup-worker-test',io.BytesIO(b'x'),1,'application/octet-stream'); print('{}')") + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + + "','attachments/cleanup-worker-test',clock_timestamp()-interval '2 hours')") + compose("start", "cleanup") + deadline = time.monotonic() + 40 + while sql("SELECT count(*) FROM attachment_uploads WHERE object_key='attachments/cleanup-worker-test'") != "0": + assert time.monotonic() < deadline, "automatic cleanup not running" + time.sleep(0.2) + assert stat('attachments/cleanup-worker-test').get("error") == "NoSuchKey" + result["compensation"]["automatic_worker"] = True + + # 10 одновременно отправляемых реальных PNG ~5.5 MiB; монитор cgroup каждые 20 ms. + image = Image.frombytes("RGB", (1600, 1200), os.urandom(1600 * 1200 * 3)) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + image.close() + data = buffer.getvalue() + expected_sha = hashlib.sha256(data).hexdigest() + monitor_code = """ +from pathlib import Path +import json,sys,threading,time +root=Path('/sys/fs/cgroup'); stop=threading.Event(); samples=[] +threading.Thread(target=lambda:(sys.stdin.readline(),stop.set()),daemon=True).start() +start=time.monotonic(); before=int((root/'memory.current').read_text()) +print('ready',flush=True) +while not stop.is_set(): + samples.append([round(time.monotonic()-start,3),int((root/'memory.current').read_text())]) + stop.wait(.02) +print(json.dumps({'baseline_bytes':before,'sample_peak_bytes':max(s[1] for s in samples), + 'lifetime_peak_bytes':int((root/'memory.peak').read_text()),'samples':samples}),flush=True) +""" + monitors = [] + responses = [] + try: + for info in apis(): + process = subprocess.Popen(["docker", "exec", "-i", info["Id"], "python", "-u", "-c", monitor_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + assert process.stdout.readline().strip() == "ready" + monitors.append((info, process)) + barrier = threading.Barrier(10) + def send(_): + concurrent_client = Client(base, timeout=60) + barrier.wait(timeout=15) + return concurrent_client.call("POST", upload_path, 201, actor=alice, raw=data, + headers={"Content-Type":"image/png"}, schema="Attachment") + started = time.monotonic() + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(send, range(10))) + elapsed = time.monotonic() - started + finally: + memory = [] + for info, process in monitors: + output, error = process.communicate("\n", timeout=10) + assert process.returncode == 0, error + measurement = json.loads(output) + measurement.update({"instance": info["Config"]["Hostname"], "limit_bytes": info["HostConfig"]["Memory"]}) + assert measurement["sample_peak_bytes"] < measurement["limit_bytes"] + memory.append(measurement) + assert len(responses) == 10 + assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2 + for response in responses: + assert response["json"]["sha256"] == expected_sha and response["json"]["size_bytes"] == len(data) + original = client.call("GET", "/api/v1/attachments/" + response["json"]["id"] + "/content", actor=bob) + assert hashlib.sha256(original["bytes"]).hexdigest() == expected_sha + assert all(not i["State"]["OOMKilled"] for i in apis()) + timings = sorted(r["elapsed"] * 1000 for r in responses) + result["parallel_uploads"] = {"concurrency":10, "successful":10, "errors":0, + "dimensions":[1600,1200], "size_bytes":len(data), "sha256":expected_sha, + "duration_seconds":round(elapsed,3), "successful_rps":round(10/elapsed,3), + "p50_ms":round(timings[4],3), "p95_ms":round(timings[-1],3), + "memory_sample_interval_seconds":0.02, "memory_includes_monitor_process":True, + "memory":memory, "all_original_checksums_verified":True, "oom_killed":False} + print("PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks:", + [m["sample_peak_bytes"] for m in memory], flush=True) + return result + finally: + compose("start", "s3", "api", "cleanup") diff --git a/student-tests/docker_lifecycle.py b/student-tests/docker_lifecycle.py new file mode 100644 index 00000000..4800ec62 --- /dev/null +++ b/student-tests/docker_lifecycle.py @@ -0,0 +1,319 @@ +"""Проверки уровня 4 в отдельном Compose project с синтетическими данными.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +import platform +import socket +import subprocess +import sys +import threading +import time +from urllib.error import HTTPError +from urllib.request import urlopen +from uuid import uuid4 + +from helpers import Client, ROOT +from client import Failure +from resilience import history +from attachment_checks import run_checks +from postgres_pause import check_postgres_pause + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True, timeout=180).strip() + + +def main(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + project = "lab4-check-" + uuid4().hex[:10] + base = f"http://127.0.0.1:{port}" + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), + API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base) + result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project, + "base_url": base, "host": platform.system() + " " + platform.machine(), + "docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"), + "commit": docker("compose", "version", "--short"), + "profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}} + result["compose_version"] = result.pop("commit") + result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + result["working_tree"] = True + + def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() + + def script(name): + subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240) + + def inspect_all(): + ids = compose("ps", "-a", "-q").split() + return json.loads(docker("inspect", *ids)) + + def apis(): + return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"] + + def metadata(info): + return {"id": info["Id"], "instance": info["Config"]["Hostname"], + "service": info["Config"]["Labels"]["com.docker.compose.service"], + "image": info["Image"], "started_at": info["State"]["StartedAt"], + "running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"], + "memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"], + "published_ports": info["HostConfig"]["PortBindings"], + "mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]} + + def sql(query): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query) + + def check_shutdown(): + info = apis()[0] + # Внутреннее подключение через docker exec не публикует порт API на хост. + pending_code = """import socket,time +s=socket.create_connection(('127.0.0.1',8080),timeout=2) +s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n') +assert b'100 Continue' in s.recv(1024) +s.sendall(b'{"username":') +print('pending',flush=True) +time.sleep(30) +""" + pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert pending.stdout.readline().strip() == "pending" + started = time.monotonic() + docker("kill", "--signal=TERM", info["Id"]) + first_refusal = None + while True: + state = json.loads(docker("inspect", info["Id"]))[0]["State"] + if not state["Running"]: + break + assert time.monotonic() - started < 8 + probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c", + "import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"], + capture_output=True, timeout=3) + if probe.returncode and first_refusal is None: + first_refusal = time.monotonic() - started + time.sleep(0.1) + elapsed = time.monotonic() - started + assert first_refusal is not None and first_refusal < elapsed < 8 + assert state["ExitCode"] == 0 and not state["OOMKilled"] + logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) + assert "Application shutdown complete" in logs.stdout + logs.stderr + result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True, + "shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3), + "exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]} + print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True) + finally: + pending.communicate(timeout=10) + compose("start", "api") + script("wait-ready.sh") + + def load_action(label, action, target=None): + client = Client(base, timeout=2) + alice, bob, _, chat = client.bootstrap() + original = client.message(alice, chat, "До воздействия")["json"] + stop = threading.Event() + lock = threading.Lock() + successes, errors, confirmed = [], [], {} + started = time.monotonic() + + def worker(index): + c = Client(base, timeout=2) + key = uuid4().hex + while not stop.is_set(): + for method in ("POST", "GET"): + try: + if method == "POST": + response = c.message(alice, chat, "load-" + key, key=key) + with lock: + confirmed[response["json"]["id"]] = response["json"] + key = uuid4().hex + else: + response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, schema="MessagePage") + with lock: + successes.append((time.monotonic() - started, method, + response["headers"]["X-Instance-Id"], response["elapsed"])) + except Failure as exc: + with lock: + errors.append((time.monotonic() - started, method, str(exc))) + stop.wait(0.05) + + with ThreadPoolExecutor(max_workers=4) as pool: + futures = [pool.submit(worker, i) for i in range(4)] + time.sleep(1) + action_start = time.monotonic() - started + try: + action_info = action() + action_end = time.monotonic() - started + # Охватывает обнаружение отказа и устойчивую работу после него. + time.sleep(max(3, 15 - (action_end - action_start))) + finally: + stop.set() + for future in futures: + future.result() + elapsed = time.monotonic() - started + items = history(client, bob, chat) + by_id = {m["id"]: m for m in items} + assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль" + assert by_id.get(original["id"]) == original + assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения" + after = [s for s in successes if s[0] >= action_end] + assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия" + recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3) + for method in ("POST", "GET")} + assert max(recovery.values()) <= 10, recovery + max_gap = {} + for method in ("POST", "GET"): + moments = sorted(s[0] for s in successes if s[1] == method) + max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3) + assert max(max_gap.values()) <= 10, max_gap + if target: + assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает" + buckets = [] + for second in range(int(elapsed) + 1): + buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes), + "errors": sum(second <= e[0] < second + 1 for e in errors)}) + latencies = sorted(s[3] * 1000 for s in successes) + summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes), + "successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2), + "first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap, + "action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3), + "p50_ms": round(latencies[len(latencies) // 2], 3), + "p95_ms": round(latencies[int(len(latencies) * .95)], 3), + "confirmed_data_preserved": True, "duplicate_messages": 0, + "instances": sorted({s[2] for s in successes}), "per_second": buckets, + "error_samples": errors[:5], "action": action_info} + print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True) + return summary + + print("Command: ./scripts/check-docker.sh; project:", project, flush=True) + print("Docker:", result["docker"], flush=True) + try: + script("start.sh") + initial = inspect_all() + result["containers"] = [metadata(info) for info in initial] + assert len(apis()) == 2 + for info in initial: + assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0 + if info["Config"]["Labels"]["com.docker.compose.service"] != "lb": + assert not info["HostConfig"]["PortBindings"] + for info in apis(): + assert docker("exec", info["Id"], "id", "-u") == "10001" + assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"]) + print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True) + subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240) + result["api_tests"] = "public smoke + student unittest suite passed" + subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30) + assert sql("SELECT count(*) FROM schema_migrations") == "3" + count = sql("SELECT count(*) FROM messages") + for _ in range(2): + compose("run", "--rm", "migrate") + assert sql("SELECT count(*) FROM schema_migrations") == "3" + assert sql("SELECT count(*) FROM messages") == count + print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True) + result["attachments"] = run_checks(base, env, compose, apis, script, sql) + + replay_client = Client(base) + replay_actor, _, _, replay_chat = replay_client.bootstrap() + replay_key = uuid4().hex + saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] + before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"], + cwd=ROOT, env=env, check=True, timeout=180) + after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + assert before.keys() == after.keys() + assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before) + assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message + result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True, + "idempotency_replay_after_restart": True} + print("PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved", flush=True) + + check_shutdown() + + dependency_client = Client(base) + actor = dependency_client.user("deps") + result["dependency_readiness"] = [] + for dependency in ("valkey", "postgres", "s3"): + try: + compose("stop", "--timeout", "8", dependency) + probe_id = apis()[0]["Id"] + readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10) + assert readiness.returncode == 1 + live = docker("exec", probe_id, "python", "-c", + "from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)") + assert live == "200" + deadline = time.monotonic() + 8 + while True: + try: + urlopen(base + "/health/ready", timeout=2).close() + except HTTPError as error: + if error.code == 503: + break + raise + except OSError: + # Уже отправленный запрос может истечь до исключения backend. + pass + assert time.monotonic() < deadline + time.sleep(0.1) + result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503}) + print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True) + finally: + compose("start", dependency) + script("wait-ready.sh") + dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + + result["postgres_pause"] = check_postgres_pause(compose, apis, base) + + subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"], + cwd=ROOT, env=env, check=True, timeout=90) + assert sum(i["State"]["Running"] for i in apis()) == 1 + result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]} + compose("start", "api") + script("wait-ready.sh") + + result["failovers"] = [] + for info in apis(): + target = info["Config"]["Hostname"] + + def stop_target(): + subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10) + stopped = json.loads(docker("inspect", info["Id"]))[0] + assert not stopped["State"]["Running"] + assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0 + # Uvicorn пишет в stderr; получаем оба потока без вывода запросов. + logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) + assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr + return metadata(stopped) + + result["failovers"].append(load_action("stop " + target, stop_target, target)) + compose("start", "api") + script("wait-ready.sh") + + before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + + def replace(): + script("rolling-update.sh") + after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + old_api = {i["id"] for i in before.values() if i["service"] == "api"} + new_api = {i["id"] for i in after.values() if i["service"] == "api"} + assert len(new_api) == 2 and not old_api & new_api + for cid, info in before.items(): + if info["service"] != "api": + assert after[cid]["started_at"] == info["started_at"] + return {"before": list(before.values()), "after": list(after.values())} + + result["rolling_update"] = load_action("sequential API replacement", replace) + result["passed"] = True + print("PASS: all lab4 grade-4 checks completed", flush=True) + finally: + output = ROOT / "evidence" / "lab4-checks.json" + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + compose("down", "--volumes", "--timeout", "8") + print("Removed only temporary test containers and volumes:", project, flush=True) + + +if __name__ == "__main__": + main() diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..d44e6fd0 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,42 @@ +from pathlib import Path +import sys +import unittest +from urllib.parse import urlencode +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client + + +class ApiTestCase(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = "/api/v1/chats/" + self.chat["id"] + + def assert_error(self, method, path, status, code, **kwargs): + if method == "POST" and path.endswith("/messages"): + kwargs["headers"] = {"Idempotency-Key": uuid4().hex, **kwargs.get("headers", {})} + result = self.client.call(method, path, status, **kwargs) + self.assertEqual(result["json"]["error"]["code"], code) + return result["json"]["error"] + + def read_all(self, path, actor, schema, limit=3): + items = [] + cursor = None + seen_cursors = set() + while True: + query = {"limit": limit} + if cursor is not None: + query["cursor"] = cursor + page = self.client.call("GET", path + "?" + urlencode(query), actor=actor, schema=schema)["json"] + self.assertLessEqual(len(page["items"]), limit) + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + self.assertTrue(page["items"], "Пустая страница не должна вести к следующей") + self.assertNotIn(cursor, seen_cursors, "Зацикливание пагинации") + seen_cursors.add(cursor) diff --git a/student-tests/lab5_pipeline.py b/student-tests/lab5_pipeline.py new file mode 100644 index 00000000..33de97e8 --- /dev/null +++ b/student-tests/lab5_pipeline.py @@ -0,0 +1,324 @@ +"""Реальный Compose-стенд: обязательный минимум и все критерии lab5 на 4.""" +from datetime import datetime, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import socket +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +from uuid import uuid4 + +from PIL import Image + +from helpers import Client, ROOT +from verify_restart import verify as verify_restart + + +def main(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + project = 'lab5-check-' + uuid4().hex[:10] + base = 'http://127.0.0.1:' + str(port) + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base, + API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5', + JOB_LEASE_SECONDS='15', JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15') + result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh', + 'project': project, 'host': platform.system() + ' ' + platform.machine(), + 'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), + 'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5, + 'timeout_seconds': 5, 'lease_seconds': 15, 'backoff_base_seconds': 0.15}, 'checks': {}} + + def command(args, timeout=240, check=True): + return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check) + + def compose(*args, timeout=240, check=True): + return command(['docker', 'compose', *args], timeout, check) + + def sql(query): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query).stdout.strip() + + def worker_code(code, service='worker'): + return compose('exec', '-T', service, 'python', '-c', code).stdout.strip() + + def wait(predicate, seconds=60, description='condition'): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.2) + raise AssertionError('Timeout: ' + description) + + def fault(job_id, point, action='crash', service='worker'): + # docker cp позволяет установить failpoint также в остановленный publisher. + container = compose('ps', '-aq', service).stdout.strip() + with TemporaryDirectory(prefix='pipeline-fault-') as directory: + source = Path(directory) / 'fault' + source.write_text(action) + source.chmod(0o644) + command(['docker', 'cp', str(source), container + ':/tmp/pipeline-faults/' + job_id + '.' + point]) + + def hit(job_id, point, service='worker'): + probe = compose('exec', '-T', service, 'python', '-c', 'from pathlib import Path; print(Path(' + repr('/tmp/pipeline-faults/' + job_id + '.' + point + '.hit') + ').exists())', check=False) + return probe.returncode == 0 and probe.stdout.strip() == 'True' + + def job_id(att): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'") + + def objects(att): + prefix = 'attachments/' + att['id'].replace('-', '') + '/v1/' + return json.loads(worker_code('import json; from src import object_store as s; print(json.dumps(sorted(o.object_name for o in s.client.list_objects(s.BUCKET,prefix=' + repr(prefix) + ',recursive=True))))', service='cleanup')) + + def stage_exists(att): + key = sql("SELECT staging_key FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'") + return int(worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(key) + ',recursive=True)))', service='cleanup')) == 1 + + def record(name, **data): + result['checks'][name] = {'passed': True, **data} + print('PASS:', name, json.dumps(data, ensure_ascii=False), flush=True) + + def metadata(att): + return client.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json'] + + def status(att, wanted, seconds=60): + return wait(lambda: (value if (value := metadata(att))['status'] == wanted else None), seconds, wanted) + + def upload(data=None, mime='image/png'): + return client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202, + actor=alice, raw=data or png, headers={'Content-Type': mime}, schema='Attachment')['json'] + + def assert_variants(att, original): + ready = status(att, 'ready') + assert {v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'} + original_response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + assert original_response['bytes'] == original + assert ready['sha256'] == hashlib.sha256(original).hexdigest() + variants = {} + for variant in ready['variants']: + response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant['name'], actor=bob) + data = response['bytes'] + assert len(data) == variant['size_bytes'] and hashlib.sha256(data).hexdigest() == variant['sha256'] + with Image.open(io.BytesIO(data)) as image: + image.load() + assert image.size == (variant['width'], variant['height']) + assert image.format == ('WEBP' if variant['name'] == 'optimized' else 'JPEG') + assert not image.getexif() + variants[variant['name']] = {'size': list(image.size), 'bytes': len(data)} + assert len(objects(att)) == 4 + return variants + + def queue_payload(att): + return json.loads(worker_code('''import json +from src import db,broker +from src.storage import utc +db.pool.open(wait=True) +with db.connection() as c: + j=c.execute("SELECT * FROM attachment_jobs WHERE attachment_id=%s",(''' + repr(att['id']) + ''',)).fetchone() + payload={'schema_version':1,'job_id':str(j['id']),'attachment_id':str(j['attachment_id']),'type':'attachment.process.v1','pipeline_version':1,'attempt':j['attempt'],'created_at':utc(j['created_at'])} +print(json.dumps(payload)) +db.pool.close() +''')) + + def duplicate(att, count=2): + payload = queue_payload(att) + from jsonschema import Draft202012Validator, FormatChecker + Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()).validate(payload) + worker_code('import json; from src import broker; c=broker.connect(); p=' + repr(json.dumps(payload)) + '; [c.xadd(broker.STREAM,{"job":p}) for _ in range(' + str(count) + ')]') + + try: + result['docker'] = command(['docker', 'info', '--format', '{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}']).stdout.strip() + command([str(ROOT / 'scripts/start.sh')], timeout=300) + for service in ('worker', 'publisher'): + worker_code("from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)", service) + ids = compose('ps', '-aq').stdout.split() + infos = json.loads(command(['docker', 'inspect', *ids]).stdout) + result['containers'] = [{'id': info['Id'], 'service': info['Config']['Labels']['com.docker.compose.service'], + 'image': info['Config']['Image'], 'memory_bytes': info['HostConfig']['Memory'], + 'nano_cpus': info['HostConfig']['NanoCpus'], + 'mounts': [{'destination': m['Destination'], 'name': m.get('Name')} for m in info['Mounts']]} for info in infos] + assert sum(i['Config']['Labels']['com.docker.compose.service'] == 'api' for i in infos) == 2 + for info in infos: + assert info['HostConfig']['Memory'] and info['HostConfig']['NanoCpus'] + if info['Config']['Labels']['com.docker.compose.service'] != 'lb': + assert not info['HostConfig']['PortBindings'] + record('topology_and_limits', api_replicas=2, broker='Valkey Streams with AOF fsync always', worker_processes=2) + public = command(['make', 'check', 'validate', 'test', 'PYTHON=' + sys.executable]) + print(public.stdout, end='', flush=True) + record('public_smoke_and_schema') + unit = command([sys.executable, '-m', 'unittest', 'discover', '-s', 'student-tests', '-v']) + print(unit.stdout + unit.stderr, end='', flush=True) + record('student_api_regressions') + client = Client(base) + alice, bob, eve, chat = client.bootstrap() + png = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + # Worker stopped: queued already linkable; API and broker restarts preserve accepted bytes/job. + compose('stop', '--timeout', '3', 'worker') + att = upload() + assert att['status'] == 'queued' and stage_exists(att) + client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice) + msg = client.message(bob, chat, '', attachments=[att['id']])['json'] + jid = job_id(att) + wait(lambda: sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't') + messages = json.loads(worker_code('from src import broker; import json; c=broker.connect(); print(json.dumps([json.loads(fields["job"]) for _,fields in c.xrange(broker.STREAM)]))', service='publisher')) + from jsonschema import Draft202012Validator, FormatChecker + validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()) + assert messages + for payload in messages: + validator.validate(payload) + compose('restart', '--timeout', '3', 'api', 'broker') + command([str(ROOT / 'scripts/wait-ready.sh')]) + assert metadata(att)['status'] == 'queued' + compose('start', 'worker') + variants = assert_variants(att, png) + assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128] + record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants) + + # COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic. + for point in ('before_publish', 'after_publish'): + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + assert sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't' + fault(jid, point, service='publisher') + compose('start', 'publisher') + wait(lambda: hit(jid, point, 'publisher'), description=point) + assert_variants(att, png) + record('outbox_crash_' + point, job_id=jid, automatic_recovery=True) + + # Worker crash after PUT before result COMMIT and after result COMMIT before ACK. + for point in ('after_objects', 'before_ack'): + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, point) + compose('start', 'publisher') + wait(lambda: hit(jid, point), description=point) + assert_variants(att, png) + if point == 'after_objects': + assert int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")) >= 2 + snapshot = metadata(att) + duplicate(att, 4) + wait(lambda: worker_code('from src import broker; c=broker.connect(); print(c.xpending(broker.STREAM,broker.GROUP)["pending"])') == '0', seconds=45, description='pending ACK') + assert metadata(att) == snapshot and len(objects(att)) == 4 + record('worker_crash_' + point, job_id=jid, attempt=int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")), canonical_objects=4, duplicate_noop=True) + + # Transient S3 errors, jittered backoff, exhausted registry and explicit replay. + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'before_process', 'transient:5') + compose('start', 'publisher') + status(att, 'failed') + failed = metadata(att) + assert failed['error_code'] == 'retry_exhausted' + client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=bob, + body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid4().hex}) + registry = json.loads(compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'failed').stdout) + assert any(row['job_id'] == jid for row in registry) + compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid) + assert_variants(att, png) + attempt = sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") + rejected = compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid, check=False) + assert rejected.returncode != 0 + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == attempt + audit = json.loads(sql("SELECT json_agg(json_build_object('event',event,'attempt',attempt,'error_code',error_code,'at',created_at) ORDER BY id) FROM attachment_job_events WHERE job_id='" + jid + "'")) + record('bounded_retry_failed_registry_replay', job_id=jid, max_automatic_attempts=5, audit=audit, ready_replay_rejected=True) + + # Permanent error from damaged staging: one attempt, safe code, no retry. + compose('stop', '--timeout', '3', 'publisher') + invalid = upload(); jid = job_id(invalid) + key = sql("SELECT staging_key FROM attachment_jobs WHERE id='" + jid + "'") + worker_code('import io; from src import object_store as s; s.put(' + repr(key) + ',io.BytesIO(b"bad"),3,"image/png")') + compose('start', 'publisher') + status(invalid, 'failed') + assert metadata(invalid)['error_code'] == 'invalid_image' + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1' + record('permanent_error', job_id=jid, attempt=1, error_code='invalid_image') + + # Реальные timeout и MemoryError внутри ограниченного resize subprocess. + for action, error_code in [('timeout', 'processing_timeout'), ('memory', 'resource_limit')]: + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'transform', action) + compose('start', 'publisher') + assert_variants(att, png) + assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='" + error_code + "'") == '1' + record('image_subprocess_' + action, job_id=jid, timeout_seconds=5, task_memory_mb=640, recovered=True) + + # Alpha, small input, EXIF rotation and floor rounding in real HTTP downloads. + cases = [] + alpha = io.BytesIO(); Image.new('RGBA', (17, 9), (255, 0, 0, 0)).save(alpha, format='PNG') + att = upload(alpha.getvalue()) + data = assert_variants(att, alpha.getvalue()) + assert data['thumbnail']['size'] == [17, 9] and data['optimized']['size'] == [17, 9] + thumb = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + with Image.open(io.BytesIO(thumb)) as image: + assert all(channel >= 250 for channel in image.getpixel((8, 4))) + cases.append({'case': 'small_alpha_on_white', 'variants': data}) + photo = io.BytesIO(); exif = Image.Exif(); exif[274] = 6 + Image.new('RGB', (321, 201), 'orange').save(photo, format='JPEG', exif=exif) + att = upload(photo.getvalue(), 'image/jpeg') + data = assert_variants(att, photo.getvalue()) + assert data['thumbnail']['size'] == [160, 256] + cases.append({'case': 'exif_orientation_6_and_floor', 'variants': data}) + record('image_semantics', cases=cases) + + # Cleanup preserves active staging even if old; removes ready and expired failed. + compose('stop', '--timeout', '3', 'worker') + active = upload() + sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 days',updated_at=clock_timestamp()-interval '2 days' WHERE id='" + active['id'] + "'") + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '2 days' WHERE id='" + invalid['id'] + "'") + orphan_id = str(uuid4()) + orphan_key = 'attachments/staging/' + orphan_id.replace('-', '') + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + orphan_id + "','" + orphan_key + "',clock_timestamp()-interval '2 days')") + worker_code('import io; from src import object_store as s; s.put(' + repr(orphan_key) + ',io.BytesIO(b"orphan"),6,"image/png")', service='cleanup') + compose('run', '--rm', '--no-deps', 'cleanup', 'python', '-m', 'src.cleanup', '--once') + assert sql("SELECT count(*) FROM attachment_uploads WHERE id='" + orphan_id + "'") == '0' + assert worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(orphan_key) + ',recursive=True)))', service='cleanup') == '0' + + assert stage_exists(active) + assert not stage_exists(invalid) + assert not stage_exists(att) + rejected = compose('exec', '-T', 'cleanup', 'python', '-m', 'src.job_admin', 'replay', job_id(invalid), check=False) + assert rejected.returncode != 0 + compose('start', 'worker'); assert_variants(active, png) + record('staging_cleanup', active_preserved=True, orphan_removed=True, ready_removed=True, expired_failed_removed=True, expired_replay_rejected=True) + + # Broker outage doesn't force synchronous resize: outbox recovers after restart. + compose('stop', '--timeout', '3', 'broker') + att = upload() + assert att['status'] == 'queued' + client.message(alice, chat, 'Broker offline; text API still works') + compose('start', 'broker'); assert_variants(att, png) + record('broker_offline_outbox', attachment_id=att['id'], recovered=True) + + record('public_persistence', restart_proof=verify_restart(env)) + failover = command(['make', 'test-failover', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/stop-one.sh']) + print(failover.stdout, end='', flush=True) + compose('start', 'api') + command([str(ROOT / 'scripts/wait-ready.sh')]) + record('public_api_failover') + result['passed'] = True + print('PASS: all lab5 grade-4 criteria completed', flush=True) + except subprocess.CalledProcessError as error: + # Команды не содержат credentials, но runtime logs не сохраняем автоматически. + print('Command failed:', error.cmd[:4], error.returncode, flush=True) + print(error.stdout, error.stderr, flush=True) + raise + finally: + output = ROOT / 'evidence/lab5-checks.json' + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + if os.environ.get('KEEP_TEST_STACK') == '1' and not result.get('passed'): + print('Kept isolated failed test stack:', project, flush=True) + else: + compose('down', '--volumes', '--timeout', '5') + print('Removed isolated test stack:', project, flush=True) + + +if __name__ == '__main__': + main() diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 00000000..07bd2fb8 --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,145 @@ +"""Пауза живой PostgreSQL: прямой readiness, ограниченное SQL I/O и восстановление.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +from uuid import uuid4 + +from helpers import Client, ROOT +from client import Failure + + +def check_postgres_pause(compose, apis, base): + client = Client(base) + alice, bob, _, chat = client.bootstrap() + saved_message = client.message(alice, chat, "До паузы PostgreSQL")["json"] + before = {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + probe_id = sorted(before)[0] + + def direct_probe(path, expected): + # HTTP напрямую в API, без HAProxy и его собственных таймаутов/503. + code = """ +import json,time +from urllib.error import HTTPError +from urllib.request import urlopen +started=time.monotonic() +try: response=urlopen('http://127.0.0.1:8080/PATH',timeout=5) +except HTTPError as error: response=error +with response: + body=json.load(response) + elapsed=time.monotonic()-started + assert response.status==EXPECTED, response.status + assert response.headers['X-Request-Id'] and response.headers['X-Instance-Id'] + if response.status==503: + assert body['error']['code']=='unavailable' + assert body['error']['request_id']==response.headers['X-Request-Id'] + assert elapsed<4.5, elapsed + print(json.dumps({'status':response.status,'elapsed_seconds':round(elapsed,3)})) +""".replace("PATH", path).replace("EXPECTED", str(expected)) + completed = subprocess.run(["docker", "exec", probe_id, "python", "-c", code], + capture_output=True, text=True, timeout=8, check=True) + return json.loads(completed.stdout) + + # Соединение получено ДО pause: проверяем именно выполняющийся SQL, а не + # только timeout получения нового соединения или check callback пула. + query_code = """ +import json,sys,time +import psycopg +from src import db +db.pool.open(wait=True,timeout=10) +conn=db.pool.getconn() +conn.execute('SET statement_timeout=0') +print('connected',flush=True) +sys.stdin.readline() +started=time.monotonic() +try: + try: conn.execute('SELECT 1') + except psycopg.OperationalError: + elapsed=time.monotonic()-started + assert elapsed<4.5, elapsed + assert conn.closed, 'timed out connection was not discarded' + print(json.dumps({'elapsed_seconds':round(elapsed,3),'connection_closed':conn.closed}),flush=True) + else: raise AssertionError('paused database answered a query') +finally: + db.pool.putconn(conn) + db.pool.close() +""" + query = subprocess.Popen(["docker", "exec", "-i", probe_id, "python", "-u", "-c", query_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = {} + try: + assert query.stdout.readline().strip() == "connected" + compose("pause", "postgres") + paused_at = time.monotonic() + output, error = query.communicate("\n", timeout=10) + assert query.returncode == 0, "SQL I/O deadline failed: " + error + result["sql_without_server_timeout"] = json.loads(output) + result["direct_ready"] = direct_probe("health/ready", 503) + result["direct_live"] = direct_probe("health/live", 200) + # Повторные запросы при длительной паузе не должны оставлять занятые потоки. + with ThreadPoolExecutor(max_workers=10) as executor: + result["repeated_ready"] = list(executor.map(lambda _: direct_probe("health/ready", 503), range(10))) + # Сервер остаётся на pause не менее 12 секунд; ограничиваем именно API. + remaining = 12 - (time.monotonic() - paused_at) + if remaining > 0: + time.sleep(remaining) + result["ready_after_12_seconds"] = direct_probe("health/ready", 503) + result["live_after_repeated_requests"] = direct_probe("health/live", 200) + result["paused_seconds"] = round(time.monotonic() - paused_at, 3) + finally: + compose("unpause", "postgres") + if query.poll() is None: + query.communicate(timeout=10) + + resumed_at = time.monotonic() + while True: + try: + result["recovered_ready"] = direct_probe("health/ready", 200) + client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + messages = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]["items"] + assert saved_message in messages + break + except (subprocess.CalledProcessError, Failure): + assert time.monotonic() - resumed_at < 30, "readiness did not recover after unpause" + time.sleep(.2) + assert before == {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + result.update({"recovery_seconds":round(time.monotonic()-resumed_at,3), "api_not_restarted":True, + "existing_session_and_message_preserved":True, "passed":True}) + print("PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart", flush=True) + return result + + +def main(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + project = "lab4-pg-pause-" + uuid4().hex[:10] + base = f"http://127.0.0.1:{port}" + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), API_REPLICAS="2", BASE_URL=base) + + def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() + + def apis(): + ids = compose("ps", "-a", "-q", "api").split() + return json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) + + result = {"utc":datetime.now(timezone.utc).isoformat(), "project":project, + "command":".venv/bin/python student-tests/postgres_pause.py", "passed":False} + try: + compose("up", "-d", "--no-build", "lb") + subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=120) + result.update(check_postgres_pause(compose, apis, base)) + print(json.dumps(result, ensure_ascii=False), flush=True) + finally: + Path(ROOT / "evidence/postgres-pause.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + compose("down", "--volumes", "--timeout", "8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..08940ae1 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,164 @@ +import base64 +import json +from urllib.parse import urlencode +from uuid import uuid4 + +from helpers import ApiTestCase + + +class ApiTests(ApiTestCase): + def test_duplicate_username(self): + self.assert_error("POST", "/api/v1/users", 409, "conflict", body=self.alice["registration"]) + + def test_invalid_user_fields(self): + invalid_bodies = [ + {}, [], None, + {"username": "ab", "display_name": "Имя"}, + {"username": "UPPER", "display_name": "Имя"}, + {"username": "name\n", "display_name": "Имя"}, + {"username": "язык", "display_name": "Имя"}, + {"username": "a" * 33, "display_name": "Имя"}, + {"username": 123, "display_name": "Имя"}, + {"username": "valid_name", "display_name": " \t\n"}, + {"username": "valid_name", "display_name": "👋" * 101}, + {"username": "valid_name", "display_name": False}, + {"username": "valid_name", "display_name": "Имя", "extra": 1}, + ] + for body in invalid_bodies: + if isinstance(body, dict): + body = {**body, "password": "Synthetic-password-123"} + with self.subTest(body=body): + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", + raw=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + + def test_malformed_json_and_encoding(self): + invalid_bodies = [ + b'{"username":', b'', b'not JSON', b'\xff', + '{"username":"valid_name","display_name":"Имя"}'.encode("utf-16"), + b'{"username":"valid_name","display_name":"\\ud800"}', + ] + for body in invalid_bodies: + with self.subTest(body=body): + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", + raw=body, headers={"Content-Type": "application/json"}) + + def test_wrong_content_type(self): + self.assert_error("POST", "/api/v1/users", 415, "unsupported_media_type", + raw=b'{}', headers={"Content-Type": "text/plain"}) + + def test_identification_and_authentication_order(self): + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized") + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers={"X-User-Id": self.alice["id"]}) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers={"Authorization": "Bearer unknown-session"}) + self.assert_error("POST", self.path + "/messages", 401, "unauthorized", + raw=b'{', headers={"Content-Type": "application/json"}) + me = self.client.call("GET", "/api/v1/users/me", schema="User", actor=self.alice, + headers={"X-User-Id": self.bob["id"]})["json"] + self.assertEqual(me, self.alice["user"]) + + def test_invalid_chat_members(self): + invalid_members = [ + [], [self.bob["id"]], [self.alice["id"], str(uuid4())], + [self.alice["id"], self.alice["id"]], + [self.alice["id"], self.alice["id"].upper()], + [self.alice["id"], "not-a-uuid"], [True], self.alice["id"], + [self.alice["id"]] + [str(uuid4()) for _ in range(100)], + ] + for members in invalid_members: + with self.subTest(members=members): + self.assert_error("POST", "/api/v1/chats", 400, "invalid_request", actor=self.alice, + body={"title": "Чат", "member_ids": members}) + chats = self.read_all("/api/v1/chats", self.alice, "ChatPage") + self.assertEqual([chat["id"] for chat in chats], [self.chat["id"]]) + + def test_invalid_chat_title(self): + for title in ("", " \n", "a" * 101, 123, None): + with self.subTest(title=title): + self.assert_error("POST", "/api/v1/chats", 400, "invalid_request", actor=self.alice, + body={"title": title, "member_ids": [self.alice["id"]]}) + + def test_chat_access(self): + outsider = self.assert_error("GET", self.path, 404, "not_found", actor=self.eve) + missing = self.assert_error("GET", "/api/v1/chats/" + str(uuid4()), 404, "not_found", actor=self.eve) + self.assertEqual(outsider["message"], missing["message"]) + self.assert_error("GET", self.path + "/messages", 404, "not_found", actor=self.eve) + self.assert_error("POST", self.path + "/messages", 404, "not_found", actor=self.eve, body={"text": "Чужое"}) + self.assertEqual(self.read_all("/api/v1/chats", self.eve, "ChatPage"), []) + self.assert_error("GET", "/api/v1/chats/not-a-uuid", 400, "invalid_request", actor=self.alice) + + def test_invalid_message_fields(self): + for text in ("", " \t\n", "👋" * 4001, 123, True, None, []): + with self.subTest(text_type=type(text).__name__): + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", + actor=self.alice, body={"text": text}) + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", actor=self.alice, + body={"text": "Подмена автора", "sender_id": self.bob["id"]}) + self.assertEqual(self.read_all(self.path + "/messages", self.alice, "MessagePage"), []) + + def test_unicode_and_text_preservation(self): + for text in (" Привет, мир 👋\n", "👋" * 4000): + message = self.client.message(self.bob, self.chat, text)["json"] + self.assertEqual(message["text"], text) + self.assertEqual(message["sender_id"], self.bob["id"]) + items = self.read_all(self.path + "/messages", self.alice, "MessagePage") + self.assertEqual([item["text"] for item in items], [" Привет, мир 👋\n", "👋" * 4000]) + + def test_message_pagination_and_changing_limit(self): + sent = [self.client.message(self.alice, self.chat, f"Сообщение {i}")["json"] for i in range(8)] + items = self.read_all(self.path + "/messages", self.bob, "MessagePage", limit=3) + self.assertEqual(items, sorted(sent, key=lambda item: (item["created_at"], item["id"]))) + first = self.client.call("GET", self.path + "/messages?limit=1", actor=self.alice, schema="MessagePage")["json"] + query = urlencode({"limit": 100, "cursor": first["next_cursor"]}) + rest = self.client.call("GET", self.path + "/messages?" + query, actor=self.alice, schema="MessagePage")["json"] + self.assertEqual(first["items"] + rest["items"], items) + self.assertIsNone(rest["next_cursor"]) + + def test_default_page_size(self): + for i in range(51): + self.client.message(self.alice, self.chat, f"Сообщение {i}") + page = self.client.call("GET", self.path + "/messages", actor=self.alice, schema="MessagePage")["json"] + self.assertEqual(len(page["items"]), 50) + self.assertIsNotNone(page["next_cursor"]) + + def test_chat_pagination_and_cursor_owner(self): + for i in range(4): + self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, + body={"title": f"Чат {i}", "member_ids": [self.alice["id"], self.bob["id"]]}) + chats = self.read_all("/api/v1/chats", self.alice, "ChatPage", limit=2) + self.assertEqual(len(chats), 5) + self.assertEqual(len({chat["id"] for chat in chats}), 5) + self.assertEqual(chats, sorted(chats, key=lambda item: (item["created_at"], item["id"]))) + first = self.client.call("GET", "/api/v1/chats?limit=1", actor=self.alice, schema="ChatPage")["json"] + query = urlencode({"cursor": first["next_cursor"]}) + self.assert_error("GET", "/api/v1/chats?" + query, 400, "invalid_request", actor=self.bob) + + def test_invalid_limits_and_cursors(self): + bad_queries = [{"limit": value} for value in ("0", "101", "-1", "1.5", "1.0", "abc", "")] + bad_queries += [{"cursor": value} for value in ("", "not-base64!", "e30=", "W10=")] + for path in ("/api/v1/chats", self.path + "/messages"): + for query in bad_queries: + with self.subTest(path=path, query=query): + self.assert_error("GET", path + "?" + urlencode(query), 400, "invalid_request", actor=self.alice) + + def test_cursor_cannot_be_used_for_another_chat_or_changed(self): + for i in range(2): + self.client.message(self.alice, self.chat, f"Сообщение {i}") + first = self.client.call("GET", self.path + "/messages?limit=1", actor=self.alice, schema="MessagePage")["json"] + cursor = first["next_cursor"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + path = "/api/v1/chats/" + other["id"] + "/messages?" + urlencode({"cursor": cursor}) + self.assert_error("GET", path, 400, "invalid_request", actor=self.alice) + data = json.loads(base64.urlsafe_b64decode(cursor)) + data["id"] = str(uuid4()) + changed = base64.urlsafe_b64encode(json.dumps(data).encode()).decode() + self.assert_error("GET", self.path + "/messages?" + urlencode({"cursor": changed}), + 400, "invalid_request", actor=self.alice) + + def test_framework_errors_and_request_ids(self): + first = self.assert_error("GET", "/unknown", 404, "not_found") + second = self.assert_error("GET", "/unknown", 404, "not_found") + self.assertNotEqual(first["request_id"], second["request_id"]) + self.assert_error("POST", "/health/live", 405, "invalid_request") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..fe8c04e2 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,135 @@ +import hashlib +import http.client +import io +import json +import struct +from urllib.parse import urlencode, urlsplit +from uuid import uuid4 +import zlib + +from PIL import Image + +from helpers import ApiTestCase, ROOT + + +class AttachmentTests(ApiTestCase): + def upload(self, data=None, filename="photo.png", content_type="image/png", expected=202, actor=None): + if data is None: + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + return self.client.call("POST", self.path + "/attachments?" + urlencode({"filename": filename}), + expected, actor=actor or self.alice, raw=data, + headers={"Content-Type": content_type}, schema="Attachment" if expected == 202 else None) + + def test_png_and_jpeg_originals(self): + jpeg = io.BytesIO() + Image.new("RGB", (100, 60), "orange").save(jpeg, format="JPEG") + for data, mime, filename in (((ROOT / "tests/fixtures/sample.png").read_bytes(), "image/png", "Фото 👋.png"), + (jpeg.getvalue(), "image/jpeg", "photo.jpeg")): + with self.subTest(mime=mime): + attachment = self.upload(data, filename, mime)["json"] + self.assertEqual(attachment["filename"], filename) + self.assertEqual(attachment["status"], "queued") + attachment = self.client.wait_ready(self.bob, attachment) + self.assertEqual({v["name"] for v in attachment["variants"]}, {"thumbnail", "crop", "optimized"}) + self.assertIsNone(attachment["error_code"]) + self.assertEqual(attachment["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(attachment["size_bytes"], len(data)) + self.assertEqual(self.client.call("GET", "/api/v1/attachments/" + attachment["id"], + actor=self.bob, schema="Attachment")["json"], attachment) + result = self.client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=self.bob) + self.assertEqual(result["bytes"], data) + self.assertEqual(result["headers"].get_content_type(), mime) + self.assertEqual(result["headers"]["X-Content-Type-Options"], "nosniff") + + def test_corruption_and_mime_spoof(self): + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + for data, mime in ((png, "image/jpeg"), (png[:50], "image/png"), + (b"not an image", "image/png"), (b"", "image/png"), + (b"GIF89a", "image/png"), (png, "application/octet-stream")): + with self.subTest(size=len(data), mime=mime): + result = self.upload(data, content_type=mime, expected=415) + self.assertEqual(result["json"]["error"]["code"], "unsupported_media_type") + + def test_filename_rejected_before_storage(self): + for filename in ("", " ", "../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\r\n.png", "a\x7f.png", "a" * 256): + with self.subTest(filename=repr(filename)): + result = self.upload(filename=filename, expected=400) + self.assertEqual(result["json"]["error"]["code"], "invalid_request") + self.client.call("POST", self.path + "/attachments", 400, actor=self.alice, raw=b"x", + headers={"Content-Type": "image/png"}) + + def test_content_length_limit_before_body(self): + self.client.call("POST", self.path + "/attachments?filename=a.png", 413, actor=self.alice, + raw=b"x", headers={"Content-Type": "image/png", "Content-Length": str(10485761)}) + + def test_chunked_limit_without_content_length(self): + target = urlsplit(self.client.base) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=15) + try: + connection.request("POST", self.path + "/attachments?filename=large.png", + body=iter([b"x" * 65536] * 160 + [b"x"]), + headers={"Authorization": "Bearer " + self.alice["token"], "Content-Type": "image/png"}, + encode_chunked=True) + response = connection.getresponse() + self.assertEqual(response.status, 413) + self.assertEqual(json.loads(response.read())["error"]["code"], "payload_too_large") + finally: + connection.close() + + def test_pixel_limit_before_decode(self): + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + # Размеры в корректном IHDR превышают лимит; pixel data вообще не выделяется. + data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 10000, 4001, 8, 2, 0, 0, 0)) + data += chunk(b"IDAT", b"") + chunk(b"IEND", b"") + self.upload(data, expected=413) + + def test_access_and_authentication(self): + attachment = self.upload()["json"] + for suffix in ("", "/content"): + path = "/api/v1/attachments/" + attachment["id"] + suffix + self.assert_error("GET", path, 401, "unauthorized") + self.assert_error("GET", path, 404, "not_found", actor=self.eve) + self.assert_error("GET", "/api/v1/attachments/" + str(uuid4()) + suffix, + 404, "not_found", actor=self.eve) + self.upload(expected=404, actor=self.eve) + self.client.call("POST", self.path + "/attachments?filename=../bad", 401, + raw=b"x", headers={"Content-Type": "image/png"}) + self.client.wait_ready(self.alice, attachment) + self.assert_error("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=missing", + 404, "not_found", actor=self.alice) + + def test_attachment_only_message_and_other_member(self): + attachment = self.upload()["json"] + message = self.client.message(self.bob, self.chat, "", attachments=[attachment["id"]])["json"] + self.assertEqual(message["attachment_ids"], [attachment["id"]]) + self.assertEqual(message["sender_id"], self.bob["id"]) + self.assertEqual(self.read_all(self.path + "/messages", self.alice, "MessagePage"), [message]) + + def test_foreign_and_unknown_attachment_ids(self): + attachment = self.upload()["json"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + for chat_id, ids in ((self.chat["id"], [str(uuid4())]), (other["id"], [attachment["id"]])): + self.assert_error("POST", "/api/v1/chats/" + chat_id + "/messages", 404, "not_found", actor=self.alice, + body={"text": "", "attachment_ids": ids}) + + def test_invalid_attachment_lists(self): + attachment = self.upload()["json"] + for ids in ([attachment["id"], attachment["id"].upper()], [str(uuid4()) for _ in range(11)], + ["not-a-uuid"], [True], None, "uuid"): + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", actor=self.alice, + body={"text": "Текст", "attachment_ids": ids}) + + def test_idempotency_includes_ordered_attachment_ids(self): + first, second = self.upload()["json"], self.upload()["json"] + key = uuid4().hex + ids = [first["id"], second["id"]] + saved = self.client.message(self.alice, self.chat, "", key=key, attachments=ids)["json"] + self.assertEqual(self.client.message(self.alice, self.chat, "", key=key, attachments=ids)["json"], saved) + for changed in ([second["id"], first["id"]], [first["id"]]): + self.assert_error("POST", self.path + "/messages", 409, "conflict", actor=self.alice, + body={"text": "", "attachment_ids": changed}, headers={"Idempotency-Key": key}) + key = uuid4().hex + saved = self.client.message(self.alice, self.chat, "Без файла", key=key)["json"] + self.assertEqual(self.client.message(self.alice, self.chat, "Без файла", key=key, attachments=[])["json"], saved) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..abebb619 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,47 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +from uuid import uuid4 + +from helpers import ApiTestCase, Client + + +class ConcurrencyTests(ApiTestCase): + def test_ten_clients_send_one_hundred_messages(self): + barrier = threading.Barrier(10) + + def send_messages(worker): + client = Client() + actor = self.alice if worker % 2 == 0 else self.bob + barrier.wait(timeout=10) + sent = [] + for index in range(10): + message = client.message(actor, self.chat, f"parallel-{worker}-{index}")["json"] + self.assertEqual(message["sender_id"], actor["id"]) + sent.append(message) + return sent + + with ThreadPoolExecutor(max_workers=10) as pool: + batches = list(pool.map(send_messages, range(10))) + sent = [message for batch in batches for message in batch] + history = self.read_all(self.path + "/messages", self.alice, "MessagePage", limit=17) + self.assertEqual(len(history), 100) + self.assertEqual(len({message["id"] for message in history}), 100) + self.assertEqual(history, sorted(sent, key=lambda item: (item["created_at"], item["id"]))) + self.assertEqual({message["text"] for message in history}, + {f"parallel-{worker}-{index}" for worker in range(10) for index in range(10)}) + print("PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages.") + + def test_simultaneous_registration_of_one_username(self): + username = "race_" + uuid4().hex[:20] + barrier = threading.Barrier(10) + + def register(worker): + client = Client() + barrier.wait(timeout=10) + return client.call("POST", "/api/v1/users", [201, 409], + body={"username": username, "display_name": f"Клиент {worker}", "password": "Synthetic-password-123"})["status"] + + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..a53fd30b --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,62 @@ +from concurrent.futures import ThreadPoolExecutor +import json +import threading +from uuid import uuid4 + +from helpers import ApiTestCase, Client + + +class IdempotencyTests(ApiTestCase): + def test_parallel_replay_across_replicas(self): + key = uuid4().hex + barrier = threading.Barrier(20) + + def send(_): + client = Client() + barrier.wait(timeout=10) + return client.message(self.alice, self.chat, "Одно сообщение 👋", key=key) + + with ThreadPoolExecutor(max_workers=20) as pool: + responses = list(pool.map(send, range(20))) + self.assertGreaterEqual(len({r["headers"]["X-Instance-Id"] for r in responses}), 2) + self.assertTrue(all(r["json"] == responses[0]["json"] for r in responses)) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [responses[0]["json"]]) + + def test_key_format_and_missing_key(self): + for key in (None, "", " ", "a b", "a/b", "a" * 129): + with self.subTest(key=key): + headers = {} if key is None else {"Idempotency-Key": key} + self.client.call("POST", self.path + "/messages", 400, actor=self.alice, + body={"text": "Текст"}, headers=headers) + for key in ("A", "a" * 128, "ABCdef012._:-"): + self.client.message(self.alice, self.chat, key, key=key) + + def test_conflict_and_semantic_json_replay(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "Текст 👋", key=key)["json"] + replay = self.client.call("POST", self.path + "/messages", 201, actor=self.alice, + raw=json.dumps({"text": "Текст 👋"}, ensure_ascii=True, indent=2).encode(), + headers={"Content-Type": "application/json", "Idempotency-Key": key}, + schema="Message")["json"] + self.assertEqual(first, replay) + self.assert_error("POST", self.path + "/messages", 409, "conflict", actor=self.alice, + body={"text": "Другой"}, headers={"Idempotency-Key": key}) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [first]) + + def test_scope_is_user_and_chat(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "Алиса", key=key)["json"] + second = self.client.message(self.bob, self.chat, "Боб", key=key)["json"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + third = self.client.message(self.alice, other, "Другой чат", key=key)["json"] + self.assertEqual(len({first["id"], second["id"], third["id"]}), 3) + + def test_replay_rechecks_session_and_access(self): + key = uuid4().hex + self.client.message(self.alice, self.chat, "Текст", key=key) + self.client.call("POST", self.path + "/messages", 404, actor=self.eve, + body={"text": "Текст"}, headers={"Idempotency-Key": key}) + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice) + self.client.call("POST", self.path + "/messages", 401, actor=self.alice, + body={"text": "Текст"}, headers={"Idempotency-Key": key}) diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..d95c4d4b --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,51 @@ +import base64 +from uuid import uuid4 + +from helpers import ApiTestCase + + +class SessionTests(ApiTestCase): + def basic(self, username, password): + value = base64.b64encode((username + ":" + password).encode()).decode() + return {"Authorization": "Basic " + value} + + def test_wrong_password_and_basic_only_for_login(self): + registration = self.alice["registration"] + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized", + headers=self.basic(registration["username"], "wrong-password")) + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized") + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized", + headers={"Authorization": "Basic not-base64!"}) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers=self.basic(registration["username"], registration["password"])) + + def test_logout_revokes_only_one_session(self): + registration = self.alice["registration"] + second = self.client.call("POST", "/api/v1/auth/sessions", 201, schema="Session", + headers=self.basic(registration["username"], registration["password"]))["json"] + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", actor=self.alice) + self.assert_error("DELETE", "/api/v1/auth/sessions/current", 401, "unauthorized", actor=self.alice) + self.client.call("GET", "/api/v1/users/me", schema="User", headers={"Authorization": "Bearer " + second["token"]}) + + def test_password_validation(self): + for password in ("short", "x" * 129, 123, None, " " * 12): + body = {"username": "pw_" + uuid4().hex[:20], "display_name": "Пароль", "password": password} + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_unicode_password_with_colon_and_private_response(self): + username = "pw_" + uuid4().hex[:20] + password = "Пароль:с:двоеточием-123" + user = self.client.call("POST", "/api/v1/users", 201, schema="User", + body={"username": username, "display_name": "Пароль", "password": password})["json"] + session = self.client.call("POST", "/api/v1/auth/sessions", 201, schema="Session", + headers=self.basic(username, password))["json"] + self.assertEqual(session["user"], user) + self.assertEqual(set(user), {"id", "username", "display_name", "created_at"}) + self.assertGreaterEqual(len(session["token"]), 32) + + def test_sql_like_message_is_plain_text(self): + text = "'; DROP TABLE messages; -- Привет" + message = self.client.message(self.alice, self.chat, text)["json"] + self.assertEqual(message["text"], text) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [message]) diff --git a/student-tests/verify_restart.py b/student-tests/verify_restart.py new file mode 100644 index 00000000..2e891a1a --- /dev/null +++ b/student-tests/verify_restart.py @@ -0,0 +1,36 @@ +"""Публичный persistence + доказательство реального restart каждого постоянного сервиса.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def verify(env): + def run(*args): + return subprocess.check_output(args, cwd=ROOT, env=env, text=True, timeout=180).strip() + def snapshot(): + ids = run('docker', 'compose', 'ps', '-q').split() + infos = json.loads(run('docker', 'inspect', *ids)) + return {info['Id']: {'service': info['Config']['Labels']['com.docker.compose.service'], + 'started_at': info['State']['StartedAt'], + 'volumes': sorted(m['Name'] for m in info['Mounts'] if m['Type'] == 'volume')} + for info in infos} + before = snapshot() + print(run('make', 'test-persistence', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/restart.sh'), flush=True) + after = snapshot() + assert before.keys() == after.keys() + assert all(before[key]['started_at'] != after[key]['started_at'] for key in before) + assert all(before[key]['volumes'] == after[key]['volumes'] for key in before) + required = {'api', 'lb', 'postgres', 'valkey', 'broker', 's3', 's3-storage', 'cleanup', 'publisher', 'worker'} + assert required == {value['service'] for value in after.values()} + return {'passed': True, 'before': before, 'after': after, 'same_containers_and_volumes': True} + + +if __name__ == '__main__': + result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': '.venv/bin/python student-tests/verify_restart.py', **verify(dict(os.environ))} + (ROOT / 'evidence/lab5-restart.json').write_text(json.dumps(result, indent=2) + '\n') + print('PASS: all persistent services actually restarted; container IDs and volumes preserved') -- 2.54.0 From fd501cbf68c29827d92bf9cf3ac4232dbc5f3ceb Mon Sep 17 00:00:00 2001 From: Lunatuk Date: Fri, 20 Nov 2026 14:00:53 +0300 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=D0=BF=D1=80=D0=BE=D0=B4=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=B0=D1=80=D0=B5=D0=BD=D0=B4=D1=8B=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B4=D0=B0=D1=87=20=D0=B8=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=82=D1=80=D0=BE=D0=BB=D1=8C=20=D0=B7=D0=B0=D0=B2=D0=B8=D1=81?= =?UTF-8?q?=D1=88=D0=B5=D0=B9=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Добавлен heartbeat для продления lease долгих задач. - Добавлен общий таймаут с остановкой дерева процессов. - Сохранена защита от устаревших исполнителей. - Добавлены тесты и обновлена документация. --- .env.example | 3 + PROJECT_MAP.md | 154 ++++++++++++++++++++++++++ REPORT.md | 13 ++- compose.yaml | 1 + docs/schema.md | 2 +- src/faults.py | 2 + src/job_runner.py | 31 ++++++ src/jobs.py | 30 ++++- src/task_supervisor.py | 69 ++++++++++++ src/worker.py | 20 +++- student-tests/lab5_pipeline.py | 40 ++++++- student-tests/test_task_supervisor.py | 88 +++++++++++++++ 12 files changed, 441 insertions(+), 12 deletions(-) create mode 100644 PROJECT_MAP.md create mode 100644 src/job_runner.py create mode 100644 src/task_supervisor.py create mode 100644 student-tests/test_task_supervisor.py diff --git a/.env.example b/.env.example index 81936309..ebe2c1fa 100644 --- a/.env.example +++ b/.env.example @@ -26,5 +26,8 @@ JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 JOB_LEASE_SECONDS=90 +# Lease продлевается каждые min(10, lease/3) секунд. Общий timeout независим. +# Для долгих задач JOB_TIMEOUT_SECONDS может быть больше JOB_LEASE_SECONDS. +JOB_TASK_TIMEOUT_SECONDS=180 JOB_BACKOFF_SECONDS=2 WORKER_TASK_MEMORY_MB=640 diff --git a/PROJECT_MAP.md b/PROJECT_MAP.md new file mode 100644 index 00000000..105421ba --- /dev/null +++ b/PROJECT_MAP.md @@ -0,0 +1,154 @@ +# Карта проекта для подготовки к защите + +Составлена по исходному коду 13.11.2026. Ссылки вида `src/jobs.py:29` обозначают файл и начальную строку функции; при изменении исходников номера нужно перепроверять. + +## Что это + +Учебный HTTP-мессенджер на Python/FastAPI. Лабораторная 5 добавляет очередь и асинхронную обработку изображений; в REPORT.md заявлена оценка 4. Возможности предыдущих работ сохранены: пользователи, сессии, чаты, сообщения, вложения, пагинация и идемпотентность сообщений. + +README.md описывает исходное задание и называет репозиторий skeleton. Фактическая реализация уже есть в src/ и compose.yaml. При ответах о поведении ориентироваться прежде всего на код. Ссылки в отчёте на REPORT.lab2/3/4.md не означают, что эти файлы присутствуют в текущем дереве. + +```text +Клиент → HAProxy → FastAPI ×2 → PostgreSQL (данные, jobs, outbox) + → Valkey (сессии) + → SeaweedFS S3 (staging) +PostgreSQL outbox → publisher → отдельный Valkey Streams broker + ↓ + worker ×2 процесса + ↓ + subprocess Pillow → S3 результаты + ↓ + PostgreSQL ready → ACK +cleanup → незавершённые загрузки и ненужный staging +``` + +## Как запускается проект + +Команда из корня в Bash/WSL/Git Bash при доступном Docker Compose: `./scripts/start.sh`. + +1. `scripts/start.sh:5`: вызывает `python3 scripts/init_local.py`. +2. `scripts/init_local.py:12`: создаёт отсутствующие .env и secrets, сохраняет существующие значения; генерирует конфигурацию доступа к S3. +3. `scripts/start.sh:6`: `docker compose up -d --build lb cleanup publisher worker`. Остальные сервисы поднимаются через depends_on. +4. `compose.yaml:90`: API собирается из Dockerfile, по умолчанию две реплики. PostgreSQL/сессии/миграции/S3/bootstrap перечислены в зависимостях API. +5. `compose.yaml:107` → `src/migrate.py:20`: отдельный migrate применяет SQL по порядку, под advisory lock, запоминает SHA-256. `src/migrate.py:11`: API только проверяет применённую схему. +6. `compose.yaml:144` → `src/bootstrap_s3.py:13`: подготовка приватного bucket. +7. `Dockerfile:28`: команда контейнера API `python -m src`. +8. `src/__main__.py:8`: Uvicorn загружает `src.app:app`, host/port из окружения, один процесс на контейнер. +9. `src/app.py:24`: lifespan открывает пул БД и проверяет миграции; `src/app.py:33` создаёт FastAPI. +10. `compose.yaml:169`: HAProxy публикуется на 127.0.0.1:8080; `infra/haproxy.cfg:34` — backend с roundrobin и проверкой readiness. +11. `scripts/start.sh:7` → `scripts/wait-ready.sh:5`: ожидание стенда через `scripts/replicas.py wait`. + +Остановка: `./scripts/stop.sh`; перезапуск: `./scripts/restart.sh`. Worker: `scripts/stop-worker.sh`, `scripts/start-worker.sh`. Остановка контейнеров с сохранением volumes отличается от удаления volumes. + +## Общий путь HTTP-запроса + +- `src/app.py:38`: middleware создаёт request_id, определяет пользователя (Basic для входа, Bearer для остальных защищённых API), проверяет JSON, добавляет X-Request-Id и X-Instance-Id, пишет время/статус в лог. +- `src/models.py:8`: Pydantic-модели и проверки входных данных; модели пользователя/чата/сообщения начинаются со строк 21/27/40. +- `src/app.py:77`: обработчики ошибок; неверные параметры дают контрактный 400, вместо стандартного FastAPI 422. +- `src/errors.py:7`: ApiError; `src/errors.py:16`: общий формат HTTP-ошибки. +- `src/db.py:28`: пул соединений; `src/db.py:42`: контекст отдельной транзакции, COMMIT при успешном выходе, rollback при ошибке. + +## Регистрация, вход, сессия + +- Регистрация: `src/app.py:112` → `src/passwords.py:12` (Argon2id) → `src/storage.py:35` (INSERT users; занятый username — 409). +- Вход: middleware `src/app.py:38` → `src/auth.py:44` (Basic username/password) → `src/app.py:180` → `src/auth.py:61` (случайный Bearer token, запись в Valkey с TTL). +- В Valkey ключ `session:SHA256(token)`, значение — user_id/expires_at: `src/auth.py:57`. Default TTL — 3600 с: `src/config.py:11`. +- Защищённый запрос: `src/auth.py:70` читает сессию, проверяет срок и существование пользователя. +- Выход: `src/app.py:185` → `src/auth.py:85`, удаление сессии. +- Valkey сессий и Valkey broker — разные сервисы: `compose.yaml:38` и `compose.yaml:50`. + +## Чаты, сообщения и доступ + +- Создание чата: `src/app.py:122` → `src/storage.py:57`, чат и участники в одной транзакции. +- Проверка членства: `src/storage.py:73`. Для недоступного чата возвращается 404. +- Создание сообщения: `src/app.py:138` проверяет Idempotency-Key → `src/storage.py:106` проверяет доступ, вложения, сохраняет сообщение и результат повтора. +- Идемпотентность сообщений: advisory lock по пользователю/пути чата/ключу, таблица message_idempotency. Тот же ключ и тело возвращают прежнее сообщение; другое тело — 409. TTL ≥24 ч: `src/config.py:18`. +- queued/processing можно прикреплять к сообщению; failed — 409: `src/storage.py:106`. +- История: `src/app.py:175` → `src/storage.py:144`. Чаты: `src/storage.py:87`. +- Keyset-пагинация по (created_at,id), LIMIT size+1; cursor содержит scope/дату/UUID: `src/pagination.py:12`, `src/pagination.py:33`. Это не OFFSET и не snapshot между страницами. + +## Загрузка изображения и ответ 202 + +`src/app.py:146` → `src/attachments.py:99` → `src/attachments.py:73`. + +1. Проверка членства, имени, MIME; поток сырых PNG/JPEG, не multipart. +2. Ограничения: до 10 MiB, до 40 млн пикселей; фактический формат, verify и полное декодирование: `src/attachments.py:19`, `src/attachments.py:47`. API проверяет изображение, но не генерирует варианты. +3. Временный файл и SHA-256; семафор ограничивает приём/проверку одной загрузкой на API-реплику. +4. Отдельно фиксируется attachment_uploads — журнал до S3 PUT. +5. Под advisory lock выполняется PUT в `attachments/staging/`. +6. В одной транзакции: attachments(status=queued), attachment_jobs, attachment_outbox; удаление строки журнала. +7. Ответ 202 после завершённого PUT и COMMIT. Возвращается снимок queued, даже если worker уже успел закончить. + +Важная граница: 202 означает устойчивый приём и сохранённое намерение обработки; готовность проверяется отдельно. В PostgreSQL хранятся метаданные, байты — в S3. TemporaryFile не является durable staging. + +## Outbox и брокер + +- `src/publisher.py:8`: выбирает доступную неопубликованную запись outbox с FOR UPDATE SKIP LOCKED → XADD → published_at → COMMIT. +- Сбой после COMMIT upload до публикации: запись остаётся в outbox. Сбой после XADD до COMMIT publisher: возможен повтор XADD. +- `src/broker.py:26`: JSON содержит ID, версию, attempt, время; без байтов, паролей и клиентских путей. +- `src/broker.py:18`: XGROUP CREATE MKSTREAM с началом 0, stream attachments:v1, группа image-workers. +- `compose.yaml:50`: AOF appendfsync always, volume broker_data, noeviction. +- `src/broker.py:33`: ACK и XDEL в одной транзакции брокера; здесь только одна consumer group. + +## Worker, состояния, защита от дублей + +- `src/worker.py:166`: supervisor запускает до двух consumer-процессов и заменяет умершие. +- `src/worker.py:140`: XAUTOCLAIM старых pending, затем XREADGROUP новых, count=1 (prefetch=1). +- `src/worker.py:89`: валидация сообщения → `src/jobs.py:29` (claim). +- Claim блокирует job и attachment, проверяет версию/attempt/статус, выставляет lease_token/lease_until и processing. +- `src/worker.py:40`: читает staging, сверяет размер/SHA-256, запускает отдельный subprocess Pillow с timeout. +- Перед финальными PUT повторно блокирует строки, проверяет lease_token и срок аренды. Старый исполнитель не публикует результат после потери аренды. +- Ключи детерминированы: `attachments//v1/{original,thumbnail,crop,optimized}`. Повтор PUT пишет в те же ключи. +- После записи всех объектов одна транзакция фиксирует ready, variants и audit; затем ACK. +- Дубль ready: `src/worker.py:28` проверяет состав вариантов и размеры объектов, после чего ACK; обработки заново нет. +- Состояния: queued → processing → ready; временная ошибка → queued; постоянная ошибка/исчерпанный бюджет → failed. +- Это at-least-once доставка и идемпотентный эффект. XADD/PUT могут повторяться; exactly-once доставки нет. + +## Retry, зависшие работы, replay + +- Исправление 13.11.2026: `src/jobs.py:51` продлевает действующий lease по токену. Consumer вызывает heartbeat каждые min(10, lease/3) секунд через `src/task_supervisor.py:49` и выполняет работу в `src/job_runner.py:9`. +- Срок аренды не ограничивает общую длительность: разрешено TIMEOUT > LEASE. Независимый JOB_TASK_TIMEOUT_SECONDS (default 180 с) останавливает всё дерево задачи при зависании; ошибка продления/потеря аренды тоже останавливает обработку. Resize timeout остаётся отдельным. +- Локальные сценарии supervisor: `student-tests/test_task_supervisor.py`. Полный Docker-прогон новой версии ещё требуется; старые evidence не подтверждают heartbeat. +- Defaults: 5 попыток, timeout subprocess 60 с, lease 90 с, backoff base 2 с: `src/jobs.py:8`. +- `src/jobs.py:72`: exponential backoff с jitter 0.5–1.5; базовая экспонента ограничена 60 с до jitter. Постоянная ошибка сразу failed, исчерпание попыток — retry_exhausted. +- `src/jobs.py:91`: отказ записывается только при совпадении lease_token. +- `src/jobs.py:98`: publisher вызывает recover для истёкших processing; причина worker_lost, ограниченный retry. +- `src/jobs.py:111`: replay только failed с неудалённым staging; тот же job_id, attempt растёт, новый ограниченный бюджет и audit. +- Реестр failed вместо отдельной DLQ: `src/job_admin.py:9`. Команды: `docker compose exec worker python -m src.job_admin failed` и `docker compose exec worker python -m src.job_admin replay `. + +## Варианты изображения + +`src/image_pipeline.py:22`: EXIF orientation, белый фон прозрачности, RGB, удаление metadata. + +| Объект | Правило | +| --- | --- | +| original | Исходные байты без перекодирования | +| thumbnail | JPEG, вписать в 256×256, пропорции сохранены, без увеличения, quality=85 | +| crop | JPEG 128×128, центрированный crop, увеличение возможно, quality=85 | +| optimized | WebP, вписать в 1280×1280, без увеличения, quality=80, method=4 | + +`src/image_pipeline.py:14`: размеры округляются вниз, минимум 1 px. `src/image_pipeline.py:58`: лимиты адресной памяти и CPU subprocess; wall timeout — `src/worker.py:40`. + +## Чтение и очистка + +- Metadata: `src/app.py:151` → `src/attachments.py:136`, доступ по членству в чате. +- Content: `src/app.py:156`: не-ready — 409; отсутствующий вариант ready — 404; StreamingResponse через API без redirect. `src/attachments.py:145` закрывает S3 response. +- `src/object_store.py:16`: приватный bucket через Minio SDK к SeaweedFS; `src/object_store.py:42` — PUT. +- `src/cleanup.py:17`: orphan-журнал старше grace period; та же advisory lock, активные загрузки пропускаются. +- `src/cleanup.py:48`: staging ready удаляется, failed хранится по retention (default сутки); queued/processing не удаляются. После удаления failed staging replay запрещён. + +## Схема, диагностика, проверки + +- `migrations/001_initial.sql`: пользователи/чаты/участники/сообщения; 002 — идемпотентность; 003 — вложения/журнал/связи; 004 — статусы/jobs/outbox/events. +- `docs/schema.md`: описание схемы и транзакционных границ. +- `src/app.py:99`: live без зависимостей; `src/app.py:104`: ready проверяет PostgreSQL, сессии и bucket. Broker/worker не входят в readiness API: намерение принимается через outbox при их простое. +- `src/config.py:7`: секреты читаются из файлов, заданных переменными *_FILE; значения секретов не выводить при подготовке. +- `make check`: структура/контракт без запуска приложения; `make validate`: OpenAPI; `make test`: smoke работающего API. +- `scripts/check-docker.sh` → `student-tests/lab5_pipeline.py:22`: полный изолированный Docker-сценарий, включая падения и восстановление. +- `student-tests/lab5_pipeline.py`: offline worker/рестарты, долгий runner с heartbeat, зависший runner с общим timeout, падение publisher/worker, retry/replay, лимиты subprocess, cleanup и offline broker. После изменений искать сценарии по комментариям и record(...). +- `src/faults.py:9`: одноразовые failpoints, включаются только ENABLE_FAULT_INJECTION=1; crash до/после публикации/эффекта/ACK, timeout/memory/transient. +- `evidence/lab5-checks.json`, `evidence/lab5-checks.txt`, `evidence/lab5-restart.json`: сохранённые результаты прошлых запусков, не подтверждение текущего живого стенда. + +## Формат быстрых ответов + +Сначала прямой ответ на вопрос в 1–3 предложениях. Ниже короткое пояснение и 1–3 ссылки на соответствующие строки кода. Термины объяснять по необходимости. По запросу о запуске показывать цепочку start.sh → Compose → Dockerfile → __main__.py → app/lifespan. При изменениях проверять номера строк заново. diff --git a/REPORT.md b/REPORT.md index 54334b3e..eb22b0ef 100644 --- a/REPORT.md +++ b/REPORT.md @@ -111,7 +111,12 @@ Replay разрешён только для failed с ещё сохранённ - Optimized: WebP quality=80, method=4; вписывание в 1280×1280 без увеличения. Уменьшение байтов каждого маленького файла не обещается. - Загрузка ≤10 MiB и ≤40 млн pixels. Limit проверяется до полной распаковки; Content-Length и фактический поток проверяются независимо. Corrupt/MIME spoof — 415, bytes/pixels overflow — 413. API допускает одну проверяемую загрузку на реплику, таймаут тела 30 s и ожидания слота 20 s. - Worker concurrency=2, каждый consumer получает максимум одну доставку за раз. Для заданного Compose memory budget разрешено concurrency 1 или 2. -- Каждый resize выполняется отдельным subprocess: wall timeout 60 s, CPU limit 60/61 s, address-space limit 640 MiB (`RLIMIT_AS`). При timeout subprocess убивается; MemoryError/resource termination не убивает основной consumer и проходит bounded retry. Worker container ограничен 1536 MiB, 2 CPU и 96 PID. Lease по умолчанию 90 s и должен превышать resize timeout минимум на 5 s. +- Каждый resize выполняется отдельным subprocess: wall timeout 60 s, CPU limit 60/61 s, address-space limit 640 MiB (`RLIMIT_AS`). При timeout subprocess убивается; MemoryError/resource termination не убивает основной consumer и проходит bounded retry. Worker container ограничен 1536 MiB, 2 CPU и 96 PID. +- После исправления 13.11.2026 lease (default 90 s, минимум 15 s) продлевается consumer каждые `min(10, lease/3)` секунд. Продление проверяет статус, lease_token и срок аренды; истёкшую или чужую аренду не восстанавливает. Исправная задача может выполняться дольше первоначального lease; resize timeout и lease больше не связаны условием `lease > timeout`. +- Вся попытка, включая скачивание, resize и публикацию, выполняется отдельным job_runner. Consumer контролирует независимый `JOB_TASK_TIMEOUT_SECONDS` (default 180 s в Compose, не меньше resize timeout). При превышении убивает дерево runner + resize и фиксирует ограниченный retry. Heartbeat не работает бесконечно для зависшего runner. При потере аренды/ошибке продления дерево также останавливается; ACK без устойчивого результата не отправляется. В Linux runner получает сигнал при смерти consumer и завершает свою группу процессов. +- Во время финальной транзакции PUT/COMMIT строки удерживаются под fence; heartbeat пропускает занятую блокировку, как и recovery. Общий timeout при этом продолжает действовать. + +Если корректное преобразование может занимать 95 s, можно задать `JOB_TIMEOUT_SECONDS=120`, `JOB_LEASE_SECONDS=90`, `JOB_TASK_TIMEOUT_SECONDS=180`: lease будет продлеваться, а отдельные таймауты останутся конечными. Defaults по-прежнему разрешают resize только до 60 s; heartbeat не отменяет этот лимит. - S3 операции имеют connect timeout 1 s/read timeout 3 s, SDK retries отключены. PostgreSQL I/O, pool wait и блокировки ограничены. CPU/resize выполняется вне DB-транзакции; транзакция публикации результата включает только fencing, четыре ограниченных PUT и COMMIT. `WORKER_TASK_MEMORY_MB` ограничивает адресное пространство процесса, а Compose mem_limit — весь контейнер. Это разные ограничения; поддержка 40 млн pixels не обещает, что любой формат/ориентация уложится в память: превышение ресурсов заканчивается наблюдаемым failed после ограниченного бюджета, а не бесконечной обработкой. @@ -145,7 +150,9 @@ Python base image закреплён digest в Dockerfile; S3 image — digest ## Доказательства критериев -Проверено 12.11.2026: все **17 групп проверок** и **39 API regression tests** прошли. Docker 29.0.1, aarch64, 10 CPU, 8 217 448 448 bytes RAM Docker VM. +Проверено 12.11.2026 до добавления heartbeat: все **17 групп проверок** и **39 API regression tests** прошли. Docker 29.0.1, aarch64, 10 CPU, 8 217 448 448 bytes RAM Docker VM. Эти evidence относятся к предыдущей реализации. + +13.11.2026 после исправления локально прошли 6 проверок `python -m unittest discover -s student-tests -p test_task_supervisor.py -v` с реальными subprocess: работа дольше первой аренды, общий timeout при продолжающемся heartbeat, потеря аренды, ошибка продления, crash и убийство дочернего процесса. Полный Docker-прогон новой версии не выполнен: Docker Engine на текущей машине недоступен. В `lab5_pipeline.py` добавлены сценарии 20-секундной работы при lease=15 s без retry и зависания runner с общим timeout=25 s; они требуют повторного `./scripts/check-docker.sh`. Полный воспроизводимый запуск: `./scripts/check-docker.sh`. Исходник сценария — `student-tests/lab5_pipeline.py`. Результаты и реальные job/attachment IDs — `evidence/lab5-checks.json`, консольный вывод — `evidence/lab5-checks.txt`. Дополнительное доказательство реального restart обычного стенда с прежними container IDs и volumes — `evidence/lab5-restart.json`; воспроизводится `.venv/bin/python student-tests/verify_restart.py`. @@ -166,7 +173,7 @@ Python base image закреплён digest в Dockerfile; S3 image — digest | Сохранность после всех рестартов | make test-persistence; public_persistence | Сессия, история и original переживают рестарты с volumes | | Отказ API | make test-failover; public_api_failover | Другая реплика продолжает обслуживать подтверждённые сообщения без дублей | -Failpoints включаются только явным `ENABLE_FAULT_INJECTION=1` во временном тестовом project; по умолчанию отключены. Файлы управления доступны только через docker exec внутри контейнера, HTTP управления ими нет. `before_publish` завершает publisher после DB COMMIT намерения до XADD; `after_publish` — после XADD до фиксации outbox. `after_objects` завершает consumer после PUT четырёх объектов до COMMIT ready; `before_ack` — после COMMIT ready до XACK. Маркеры срабатывания проверяются тестом, поэтому простой быстрый успех без внедрённого сбоя не засчитывается. Отдельный timeout failpoint усыпляет именно ограниченный resize subprocess, не подменяет результат mock-объектом. +Failpoints включаются только явным `ENABLE_FAULT_INJECTION=1` во временном тестовом project; по умолчанию отключены. Файлы управления доступны только через docker exec внутри контейнера, HTTP управления ими нет. `before_publish` завершает publisher после DB COMMIT намерения до XADD; `after_publish` — после XADD до фиксации outbox. `after_objects` завершает job_runner после PUT четырёх объектов до COMMIT ready; `before_ack` — после COMMIT ready до XACK. Маркеры срабатывания проверяются тестом, поэтому простой быстрый успех без внедрённого сбоя не засчитывается. Отдельный timeout failpoint усыпляет именно ограниченный resize subprocess, не подменяет результат mock-объектом. `before_download` с `delay:20` задерживает корректную попытку, с `timeout` — имитирует зависание runner до скачивания. ## Эксперименты и ограничения diff --git a/compose.yaml b/compose.yaml index 40eb5de9..bbb4089b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,7 @@ x-application: &application JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + JOB_TASK_TIMEOUT_SECONDS: ${JOB_TASK_TIMEOUT_SECONDS:-180} JOB_BACKOFF_SECONDS: ${JOB_BACKOFF_SECONDS:-2} WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640} diff --git a/docs/schema.md b/docs/schema.md index 2e3cd364..b41d57c5 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -21,7 +21,7 @@ `attachment_jobs` хранит один job_id на attachment_id, staging_key, pipeline_version=1, текущую попытку, бюджет, lease_token/lease_until и отметку удаления staging. `attachment_outbox(job_id,attempt)` создаётся в той же транзакции, что metadata/намерение. Dispatcher отмечает published_at после XADD; потеря ответа вызывает безопасный дубль. Отложенный retry получает отдельную строку outbox с available_at. `attachment_job_events` — audit переходов и явных replay без credentials и текста исключений. -Данные при 202: приватный S3 `attachments/staging/` и COMMIT metadata+job+outbox. Готовность: worker сохраняет неизменённый original и три варианта по детерминированным ключам, затем атомарно фиксирует status=ready и JSON metadata. До ready content отвечает 409. Worker проверяет lease_token под блокировкой строк перед записью результатов. Cleaner не удаляет staging queued/processing; после ready удаляет staging, а failed хранит до STAGING_RETENTION_SECONDS. +Данные при 202: приватный S3 `attachments/staging/` и COMMIT metadata+job+outbox. Готовность: worker сохраняет неизменённый original и три варианта по детерминированным ключам, затем атомарно фиксирует status=ready и JSON metadata. До ready content отвечает 409. Worker проверяет lease_token под блокировкой строк перед записью результатов. Consumer продлевает действующую аренду каждые min(10, lease/3) секунд, пока отдельный job_runner выполняет попытку. Просроченный/чужой lease продлевать нельзя. Общий JOB_TASK_TIMEOUT_SECONDS ограничивает работу runner независимо от heartbeat: по deadline убивается всё дерево обработки. Lease может быть меньше времени допустимой обработки. Cleaner не удаляет staging queued/processing; после ready удаляет staging, а failed хранит до STAGING_RETENTION_SECONDS. Индексы: UNIQUE users(username); chat_members(user_id, chat_id); messages(chat_id, created_at, id); attachments(chat_id); attachment_uploads(created_at). Первичные/уникальные ключи индексируют поиск ID и object_key. diff --git a/src/faults.py b/src/faults.py index d0f0781d..612f8f7a 100644 --- a/src/faults.py +++ b/src/faults.py @@ -29,3 +29,5 @@ def hit(job_id, point): bytearray(1024 * 1024 * 1024) if action == 'timeout': time.sleep(300) + if action.startswith('delay:'): + time.sleep(float(action.split(':', 1)[1])) diff --git a/src/job_runner.py b/src/job_runner.py new file mode 100644 index 00000000..02f083d8 --- /dev/null +++ b/src/job_runner.py @@ -0,0 +1,31 @@ +"""Одна попытка в отдельном процессе под контролем consumer-supervisor.""" +import sys +from uuid import UUID + +from src import db, jobs, task_supervisor +from src.worker import ProcessingError, process + + +def main(): + task_supervisor.watch_parent(int(sys.argv[3])) + db.pool.open(wait=True, timeout=15) + try: + with db.connection() as conn: + job = jobs.locked(conn, UUID(sys.argv[1])) + if not job or job['status'] != 'processing' or job['lease_token'] != UUID(sys.argv[2]): + return 75 + try: + if not process(job, sys.argv[4]): + return 75 + except ProcessingError as error: + jobs.fail(job, error.code, error.permanent) + except Exception: + jobs.fail(job, 'storage_unavailable') + # Только после COMMIT ready/retry/failed родитель может отправить ACK. + return 0 + finally: + db.pool.close() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py index 360b7312..6fd5ed8f 100644 --- a/src/jobs.py +++ b/src/jobs.py @@ -8,8 +8,10 @@ from src import db MAX_ATTEMPTS = int(os.environ.get('JOB_MAX_ATTEMPTS', '5')) TIMEOUT = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60')) LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '90')) +TASK_TIMEOUT = int(os.environ.get('JOB_TASK_TIMEOUT_SECONDS', str(max(180, TIMEOUT + 60)))) +HEARTBEAT = min(10, LEASE / 3) BACKOFF = float(os.environ.get('JOB_BACKOFF_SECONDS', '2')) -if not 1 <= MAX_ATTEMPTS <= 20 or TIMEOUT < 1 or LEASE < TIMEOUT + 5 or BACKOFF <= 0: +if not 1 <= MAX_ATTEMPTS <= 20 or TIMEOUT < 1 or LEASE < 15 or TASK_TIMEOUT < TIMEOUT or BACKOFF <= 0: raise ValueError('Неверные retry/timeout/lease настройки') @@ -46,6 +48,27 @@ def claim(payload): return 'claimed', job +def renew(job): + """Продлить только действующую аренду; истёкшую/чужую не воскрешаем.""" + with db.connection() as conn: + current = conn.execute('''SELECT j.lease_token,j.lease_until,a.status + FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE j.id=%s FOR UPDATE OF j,a SKIP LOCKED''', (job['id'],)).fetchone() + if current is None: + # Финальная запись объектов держит эти строки под fence. Не ждём + # её блокировку: recover также пропустит её, общий timeout действует. + current = conn.execute('''SELECT j.lease_token,a.status FROM attachment_jobs j + JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s''', (job['id'],)).fetchone() + return bool(current and current['status'] == 'processing' + and current['lease_token'] == job['lease_token']) + if current['status'] != 'processing' or current['lease_token'] != job['lease_token']: + return False + return conn.execute('''UPDATE attachment_jobs + SET lease_until=clock_timestamp()+%s*interval '1 second' + WHERE id=%s AND lease_until>clock_timestamp() RETURNING id''', + (LEASE, job['id'])).fetchone() is not None + + def retry_or_fail(conn, job, code, permanent=False): # replay_count задаёт новый ограниченный бюджет, сохраняя глобальный номер attempt. exhausted = job['attempt'] >= job['max_attempts'] @@ -79,7 +102,10 @@ def recover(): ORDER BY j.lease_until LIMIT 20 FOR UPDATE OF j SKIP LOCKED''').fetchall() for row in rows: job = locked(conn, row['id']) - retry_or_fail(conn, job, 'worker_lost') + expired = conn.execute('''SELECT lease_until= deadline: + raise subprocess.TimeoutExpired(command, timeout) from None + if not renew(): + raise LeaseLost() + finally: + # В том числе убиваем дочерний image_pipeline, если завис его родитель. + stop_tree(process) diff --git a/src/worker.py b/src/worker.py index 5a9b66ce..26c25621 100644 --- a/src/worker.py +++ b/src/worker.py @@ -16,7 +16,7 @@ from uuid import UUID, uuid4 from psycopg.types.json import Jsonb -from src import broker, db, faults, jobs, object_store +from src import broker, db, faults, jobs, object_store, task_supervisor from src.errors import ApiError @@ -37,8 +37,9 @@ def verify_ready(job): raise ProcessingError('incomplete_result') -def process(job): - with TemporaryDirectory(prefix='image-job-') as directory: +def process(job, workspace=None): + faults.hit(job['id'], 'before_download') + with TemporaryDirectory(prefix='image-job-', dir=workspace) as directory: output = Path(directory) source = output / 'original' response = object_store.get(job['staging_key']) @@ -115,8 +116,19 @@ def handle(client, message_id, fields): if state == 'claimed': try: faults.hit(job['id'], 'before_process') - if not process(job): + # Временные файлы удаляет consumer даже после SIGKILL runner. + with TemporaryDirectory(prefix='image-attempt-') as workspace: + result = task_supervisor.run( + [sys.executable, '-m', 'src.job_runner', str(job['id']), str(job['lease_token']), + str(os.getpid()), workspace], + renew=lambda: jobs.renew(job), timeout=jobs.TASK_TIMEOUT, interval=jobs.HEARTBEAT) + if result != 0: + # Crash/неизвестный COMMIT: не ACK, восстановление через lease. return + except task_supervisor.LeaseLost: + return + except subprocess.TimeoutExpired: + jobs.fail(job, 'processing_timeout') except ProcessingError as error: jobs.fail(job, error.code, error.permanent) except Exception: diff --git a/student-tests/lab5_pipeline.py b/student-tests/lab5_pipeline.py index 33de97e8..d08c7075 100644 --- a/student-tests/lab5_pipeline.py +++ b/student-tests/lab5_pipeline.py @@ -27,12 +27,14 @@ def main(): base = 'http://127.0.0.1:' + str(port) env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base, API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5', - JOB_LEASE_SECONDS='15', JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15') + JOB_LEASE_SECONDS='15', JOB_TASK_TIMEOUT_SECONDS='25', + JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15') result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh', 'project': project, 'host': platform.system() + ' ' + platform.machine(), 'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), 'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5, - 'timeout_seconds': 5, 'lease_seconds': 15, 'backoff_base_seconds': 0.15}, 'checks': {}} + 'timeout_seconds': 5, 'task_timeout_seconds': 25, 'lease_seconds': 15, + 'backoff_base_seconds': 0.15}, 'checks': {}} def command(args, timeout=240, check=True): return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check) @@ -180,6 +182,40 @@ db.pool.close() assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128] record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants) + # Исправная попытка живёт дольше исходного lease без ложного retry. + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'before_download', 'delay:20') + compose('start', 'publisher') + wait(lambda: hit(jid, 'before_download'), description='long task started') + initial_lease = sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'") + assert worker_code('''from uuid import uuid4 +from src import db,jobs +db.pool.open(wait=True) +with db.connection() as c: + j=jobs.locked(c,''' + repr(jid) + ''') +j['lease_token']=uuid4() +print(jobs.renew(j)) +db.pool.close() +''') == 'False' + wait(lambda: sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'") != initial_lease, + seconds=12, description='heartbeat extended lease') + assert_variants(att, png) + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1' + assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND event='retry'") == '0' + record('long_task_heartbeat', job_id=jid, delay_seconds=20, lease_seconds=15, attempt=1) + + # Heartbeat не сохраняет зависшую работу бесконечно: общий deadline. + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'before_download', 'timeout') + compose('start', 'publisher') + assert_variants(att, png) + assert hit(jid, 'before_download') + assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='processing_timeout'") == '1' + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '2' + record('whole_task_timeout', job_id=jid, timeout_seconds=25, recovered=True) + # COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic. for point in ('before_publish', 'after_publish'): compose('stop', '--timeout', '3', 'publisher') diff --git a/student-tests/test_task_supervisor.py b/student-tests/test_task_supervisor.py new file mode 100644 index 00000000..36307a96 --- /dev/null +++ b/student-tests/test_task_supervisor.py @@ -0,0 +1,88 @@ +"""Локальные проверки реальных subprocess без Docker и внешних библиотек. + +python -m unittest discover -s student-tests -p test_task_supervisor.py -v +""" +from pathlib import Path +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src import task_supervisor + + +class TaskSupervisorTests(unittest.TestCase): + def test_successful_work_outlives_initial_lease(self): + lease = 0.5 + expires = time.monotonic() + lease + renewals = [] + + def renew(): + nonlocal expires + now = time.monotonic() + self.assertLess(now, expires, 'renewal must happen before lease expires') + expires = now + lease + renewals.append(now) + return True + + started = time.monotonic() + code = task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(1.1)'], + renew, timeout=5, interval=0.1) + self.assertEqual(code, 0) + self.assertGreater(time.monotonic() - started, lease) + self.assertGreaterEqual(len(renewals), 3) + + def test_heartbeat_does_not_bypass_total_timeout(self): + renewals = [] + + def renew(): + renewals.append(True) + return True + + started = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'], + renew, timeout=0.5, interval=0.1) + self.assertTrue(renewals) + self.assertLess(time.monotonic() - started, 5) + + def test_lost_lease_stops_work(self): + with self.assertRaises(task_supervisor.LeaseLost): + task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'], + lambda: False, timeout=10, interval=0.1) + + def test_renewal_error_stops_work(self): + def renew(): + raise ConnectionError('database unavailable') + + with self.assertRaises(ConnectionError): + task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'], + renew, timeout=10, interval=0.1) + + def test_child_failure_is_not_success(self): + self.assertEqual(task_supervisor.run([sys.executable, '-c', 'raise SystemExit(86)'], + lambda: True, timeout=5, interval=0.1), 86) + + def test_timeout_kills_descendant_too(self): + with TemporaryDirectory() as directory: + marker = Path(directory) / 'survived' + started = Path(directory) / 'started' + descendant = ('import time; from pathlib import Path; ' + f'Path({str(started)!r}).write_text("started"); ' + 'time.sleep(1.5); ' + f'Path({str(marker)!r}).write_text("orphan")') + parent = ('import subprocess,sys,time; ' + f'subprocess.Popen([sys.executable,"-c",{descendant!r}]); ' + 'time.sleep(60)') + with self.assertRaises(subprocess.TimeoutExpired): + task_supervisor.run([sys.executable, '-c', parent], lambda: True, + timeout=0.8, interval=0.1) + self.assertTrue(started.exists(), 'descendant must start before timeout') + time.sleep(1.6) + self.assertFalse(marker.exists(), 'descendant survived parent timeout') + + +if __name__ == '__main__': + unittest.main() -- 2.54.0