From 0b399073d6bc3d0ab7c308d466b2cd4ad21511f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9=20=D0=A7=D0=B5?= =?UTF-8?q?=D1=80=D0=BD=D0=B8=D0=BA=D0=BE=D0=B2?= Date: Wed, 11 Nov 2026 23:56:28 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=205=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22304=20=E2=80=94=20=D0=A7=D0=B5=D1=80=D0=BD=D0=B8=D0=BA?= =?UTF-8?q?=D0=BE=D0=B2=20=E2=80=94=20=D0=BD=D0=B0=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 21 + .env.example | 12 +- .gitattributes | 5 + DEMO.md | 114 +++++ Dockerfile | 41 ++ Makefile | 24 +- REPORT.md | 177 +++++++- compose.yaml | 295 ++++++++++++ evidence/README.md | 16 + evidence/cleanup.txt | 3 + evidence/failover.txt | 3 + evidence/lab5-checks.txt | 23 + evidence/offline.txt | 1 + evidence/persistence.txt | 2 + evidence/postgres-pause.txt | 10 + evidence/smoke.txt | 1 + evidence/stack.json | 182 ++++++++ evidence/stack.txt | 1 + evidence/tests.txt | 103 +++++ evidence/unit.txt | 23 + infra/README.md | 11 + infra/broker/valkey.conf | 11 + infra/haproxy/haproxy.cfg | 72 +++ infra/seaweedfs/filer.toml | 4 + infra/valkey/valkey.conf | 23 + migrations/0001_schema.sql | 46 ++ migrations/0002_messages_history_index.sql | 8 + migrations/0003_idempotency_keys.sql | 20 + migrations/0004_attachments.sql | 27 ++ migrations/0005_async_attachments.sql | 91 ++++ requirements.in | 8 + requirements.txt | 360 +++++++++++++++ scripts/check_postgres_pause.py | 208 +++++++++ scripts/explain_history.py | 139 ++++++ scripts/init_secrets.py | 69 +++ scripts/lab5_checks.py | 487 ++++++++++++++++++++ scripts/restart.cmd | 10 + scripts/restart.sh | 11 + scripts/stack_evidence.py | 35 ++ scripts/stop-one.cmd | 19 + scripts/stop-one.sh | 19 + scripts/verify.py | 86 ++++ src/messenger/__init__.py | 1 + src/messenger/__main__.py | 94 ++++ src/messenger/broker.py | 95 ++++ src/messenger/bucket.py | 31 ++ src/messenger/cleanup_objects.py | 84 ++++ src/messenger/config.py | 84 ++++ src/messenger/db.py | 309 +++++++++++++ src/messenger/dispatcher.py | 62 +++ src/messenger/errors.py | 29 ++ src/messenger/healthcheck.py | 26 ++ src/messenger/image_check.py | 29 ++ src/messenger/image_process.py | 123 +++++ src/messenger/jobs.py | 319 +++++++++++++ src/messenger/jobs_cli.py | 74 +++ src/messenger/migrate.py | 77 ++++ src/messenger/objects.py | 40 ++ src/messenger/passwords.py | 50 +++ src/messenger/server.py | 498 +++++++++++++++++++++ src/messenger/sessions.py | 81 ++++ src/messenger/timeutil.py | 18 + src/messenger/validation.py | 219 +++++++++ src/messenger/worker.py | 307 +++++++++++++ student-tests/api.py | 144 ++++++ student-tests/fixtures/README.md | 6 + student-tests/fixtures/sample.jpg | Bin 0 -> 7909 bytes student-tests/race.py | 88 ++++ student-tests/test_access.py | 87 ++++ student-tests/test_attachments.py | 188 ++++++++ student-tests/test_balancing.py | 39 ++ student-tests/test_concurrency.py | 62 +++ student-tests/test_idempotency.py | 106 +++++ student-tests/test_pagination.py | 127 ++++++ student-tests/test_sessions.py | 160 +++++++ student-tests/test_validation.py | 219 +++++++++ unit-tests/test_image_process.py | 168 +++++++ 77 files changed, 6838 insertions(+), 27 deletions(-) create mode 100644 .gitattributes create mode 100644 DEMO.md create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/cleanup.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/lab5-checks.txt create mode 100644 evidence/offline.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/postgres-pause.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stack.json create mode 100644 evidence/stack.txt create mode 100644 evidence/tests.txt create mode 100644 evidence/unit.txt create mode 100644 infra/broker/valkey.conf create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/seaweedfs/filer.toml create mode 100644 infra/valkey/valkey.conf create mode 100644 migrations/0001_schema.sql create mode 100644 migrations/0002_messages_history_index.sql create mode 100644 migrations/0003_idempotency_keys.sql create mode 100644 migrations/0004_attachments.sql create mode 100644 migrations/0005_async_attachments.sql create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 scripts/check_postgres_pause.py create mode 100644 scripts/explain_history.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/lab5_checks.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/stack_evidence.py create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/__main__.py create mode 100644 src/messenger/broker.py create mode 100644 src/messenger/bucket.py create mode 100644 src/messenger/cleanup_objects.py create mode 100644 src/messenger/config.py create mode 100644 src/messenger/db.py create mode 100644 src/messenger/dispatcher.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/healthcheck.py create mode 100644 src/messenger/image_check.py create mode 100644 src/messenger/image_process.py create mode 100644 src/messenger/jobs.py create mode 100644 src/messenger/jobs_cli.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/objects.py create mode 100644 src/messenger/passwords.py create mode 100644 src/messenger/server.py create mode 100644 src/messenger/sessions.py create mode 100644 src/messenger/timeutil.py create mode 100644 src/messenger/validation.py create mode 100644 src/messenger/worker.py create mode 100644 student-tests/api.py create mode 100644 student-tests/fixtures/README.md create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/race.py create mode 100644 student-tests/test_access.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_pagination.py create mode 100644 student-tests/test_sessions.py create mode 100644 student-tests/test_validation.py create mode 100644 unit-tests/test_image_process.py diff --git a/.dockerignore b/.dockerignore index bc16e55..be2f7ee 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,24 @@ __pycache__ node_modules evidence .DS_Store +# В образ копируются только requirements.txt, migrations/ и src/. +**/__pycache__ +**/*.pyc +contracts +examples +scripts +tests +student-tests +unit-tests +infra +*.md +Makefile +lab.json +requirements-dev.txt +requirements.in +compose*.yaml +.gitignore +.gitattributes +.vscode +.idea +*.log diff --git a/.env.example b/.env.example index 119384b..07f226e 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,11 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id @@ -19,7 +19,11 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +JOB_TIMEOUT_SECONDS=10 +JOB_LEASE_SECONDS=45 +IMAGE_MEMORY_MIB=768 +JOB_RETRY_BASE_SECONDS=1 +JOB_RETRY_MAX_SECONDS=30 STAGING_RETENTION_SECONDS=86400 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e83120d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto +*.sh text eol=lf +*.cmd text eol=crlf +*.png binary +*.jpg binary diff --git a/DEMO.md b/DEMO.md new file mode 100644 index 0000000..40eb7f9 --- /dev/null +++ b/DEMO.md @@ -0,0 +1,114 @@ +# Шпаргалка для защиты — лабораторная 5, на 4 + +Рабочая папка: `D:\VScodeproject\VNP\mtusi2026lab5`. Нужен запущенный Docker Desktop. В Windows используем `python -X utf8` для публичных скриптов. В Linux — `python3`, хукам `.sh` нужен `chmod +x scripts/*.sh`. + +## 1. Запуск + +```powershell +cd D:\VScodeproject\VNP\mtusi2026lab5 +python scripts/init_secrets.py +Copy-Item .env.example .env +docker compose up -d --build --wait +docker compose ps -a +``` + +Если `.env` уже настроен, повторно копировать пример не нужно. Секреты создаются только при отсутствии, не печатаются и не попадают в Git. Данные lab5 находятся в отдельных volumes проекта `mtusi-lab5`. + +API — http://127.0.0.1:8080, состояние API-реплик — http://127.0.0.1:8404. У API нет web-интерфейса. При занятых портах остановите прежний стенд либо настройте APP_PUBLISHED_PORT/LB_STATS_PORT. + +```powershell +docker compose logs --tail 15 worker dispatcher +``` + +В логах видны job_id, попытка, ready/retry/failed; токены и содержимое файлов не пишутся. + +## 2. Проверки API и вариантов + +```powershell +python -X utf8 scripts/verify.py +``` + +Проверяются skeleton, unit-тесты pipeline, публичный smoke, собственные тесты и фактическая топология. Upload возвращает 202 queued, затем GET достигает ready. Готовый результат содержит original без изменений, thumbnail JPEG fit256, crop JPEG128×128 и optimized WebP fit1280. Для fixture320×200 thumbnail256×160. + +Можно запустить отдельно: + +```powershell +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 -m unittest discover -s student-tests -v +python -X utf8 scripts/verify.py --only unit +``` + +## 3. Сохранность принятой работы + +```powershell +python -X utf8 scripts/lab5_checks.py --only durability +``` + +Скрипт останавливает worker, загружает файл, получает 202 queued и 409 на content, создаёт сообщение с вложением. Затем перезапускает API и broker с volumes. После запуска worker та же логическая job становится ready, оригинал и три варианта проверяются по SHA-256. + +## 4. Все дополнительные требования на 4 + +```powershell +python -X utf8 scripts/lab5_checks.py --only outbox broker_outage +python -X utf8 scripts/lab5_checks.py --only redelivery +python -X utf8 scripts/lab5_checks.py --only crash lease +python -X utf8 scripts/lab5_checks.py --only retries permanent +python -X utf8 scripts/lab5_checks.py --only staging +``` + +- Outbox: dispatcher остановлен; upload COMMIT уже прошёл; API убивается до publish. После запуска задача доходит до ready без нового задания. +- Broker outage: 202 и сообщения работают через устойчивый DB outbox, после возврата брокера работа обрабатывается. +- Redelivery: дубль ready подтверждается без изменения результата/attempt/времени готовности. +- Crash: одноразовый fault завершает worker после четырёх S3 PUT, до публикации ready и ack. Lease истекает; повтор формирует тот же набор из трёх вариантов. +- Lease: зависший processing worker убивается; новая попытка завершается, старая аренда уже не может записать ready или failed. +- Retry: временные ошибки используют backoff+jitter; после пяти попыток failed. Операторский replay сохраняет job_id и audit. Постоянная ошибка сразу failed, content/прикрепление дают409. +- Staging: queued исходник защищён, удаление staging разрешено только после COMMIT ready. Оригинал остаётся. + +Полный сбойный прогон: + +```powershell +python -X utf8 scripts/verify.py --faults +``` + +Тесты восстанавливают остановленные сервисы в finally. Сценарии crash/lease ждут реальный lease45с, поэтому занимают заметное время. Не запускайте их одновременно друг с другом. + +## 5. Операторские команды + +```powershell +docker compose --profile tools run --rm jobs failed +docker compose --profile tools run --rm jobs show JOB_UUID +docker compose --profile tools run --rm jobs replay ATTACHMENT_UUID --actor Chernikov --reason "Причина исправлена" +docker compose --profile tools run --rm jobs redeliver JOB_UUID +docker compose --profile tools run --rm jobs cleanup +``` + +UUID берутся из результата загрузки/реестра/логов. Replay ready — no-op с записью audit; только failed с сохранённым staging запускается заново. Внешнего административного API нет. + +Очистка orphan и failed staging после retention24ч: + +```powershell +docker compose --profile tools run --rm cleanup +docker compose --profile tools run --rm cleanup --apply +``` + +По умолчанию dry-run. Queued/processing защищены; неучтённые объекты моложе часа пропускаются. Failed staging удаляется только после явного durable tombstone, который запрещает дальнейший replay. + +## 6. Общий перезапуск и остановка + +```powershell +python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080 +docker compose up -d --wait +docker compose stop +``` + +Повторный запуск: `docker compose up -d --wait`. `docker compose down` сохраняет volumes. **`docker compose down -v` удаляет все учебные данные lab5**, включая БД, S3 и очередь. + +## Что объяснить преподавателю + +- В момент202 уже сохранены все bytes в S3 staging и DB-намерение обработки. 202 не означает, что resize завершён. +- Outbox закрывает промежуток между DB COMMIT и публикацией в брокер. +- XADD/долговечный приём брокера и XACK worker — разные границы подтверждения. +- Доставка at least once допускает дубль; ключ результата `(attachment_id,pipeline_version,variant)` делает эффект идемпотентным. +- Lease имеет срок, а token/attempt fencing запрещает устаревшему worker зафиксировать результат. +- Задания на 5 и исследовательские треки не заявлены. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f61465 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# Базовый образ закреплён и тегом версии, и digest: тег читается человеком, +# digest гарантирует те же самые слои при повторной сборке. +FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d + +# PYTHONUNBUFFERED: логи сразу попадают в docker logs, а не ждут в буфере. +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +# Непривилегированный пользователь без домашнего каталога и без shell. +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app + +# Сначала только lock-файл: слой с пакетами пересобирается лишь при изменении зависимостей. +# --require-hashes: pip ставит ровно те файлы, хеши которых записаны в requirements.txt. +# --only-binary: только готовые wheel, без компиляции во время сборки. +COPY requirements.txt ./ +RUN pip install --no-cache-dir --require-hashes --only-binary=:all: --root-user-action=ignore -r requirements.txt + +# Python не компилируется в отдельный бинарник, поэтому сборка одноэтапная. +# Файлы принадлежат root: процесс приложения может их читать, но не менять. +COPY migrations/ ./migrations/ +COPY src/ ./ + +# Версия сборки попадает в лог старта: по ней видно, какие реплики уже обновлены. +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +USER app +EXPOSE 8080 + +# Во время start-period Docker проверяет каждую секунду: новая реплика быстрее становится healthy. +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --start-interval=1s --retries=3 \ + CMD ["python", "-m", "messenger.healthcheck"] + +# exec-форма: python — процесс PID 1 и сам получает SIGTERM от docker stop. +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d17..2f9460e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://127.0.0.1:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -28,3 +28,25 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: secrets up student-test verify checks cleanup test-postgres-pause +secrets: + $(PYTHON) scripts/init_secrets.py + +up: secrets + docker compose up -d --build --wait + +student-test: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -v + +verify: + $(PYTHON) scripts/verify.py --base-url "$(BASE_URL)" + +checks: + $(PYTHON) scripts/lab5_checks.py --base-url "$(BASE_URL)" + +cleanup: + docker compose --profile tools run --rm cleanup + +test-postgres-pause: + $(PYTHON) scripts/check_postgres_pause.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index e98f75e..4f50f14 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,170 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор / группа: Черников Дмитрий, БВТ2304. +- Целевая оценка: **4** — обязательные требования на 3 и все пункты на 4. Треки на 5 не заявляются. +- Ветка: `solution/bvt2304-chernikov`, создана от официального skeleton `mtusi2026/mtusi2026lab5:main`, базовый commit `276dbda`. Commit решения/PR указывается после публикации в личный fork. +- Основа — собственная лабораторная 4, включая клиентский таймаут PostgreSQL 3 с из исправления по ревью. При разработке использовался Codex (OpenAI). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Compose и Python 3.9+ для HTTP-тестов. Команды выполняются из корня `mtusi2026lab5`. Сервер и worker используют Python 3.14.7 внутри образа. Точные версии и хеши библиотек находятся в `requirements.txt`: psycopg 3.3.6, psycopg-pool 3.3.3, valkey 6.1.1, argon2-cffi 25.1.0, boto3 1.43.123, Pillow 13.0.0. -## Архитектура и решения +```powershell +python scripts/init_secrets.py +Copy-Item .env.example .env +docker compose config --quiet +docker compose up -d --build --wait +docker compose ps -a +python -X utf8 scripts/verify.py +python -X utf8 scripts/lab5_checks.py +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +В Linux используйте `python3`, вместо Copy-Item — `cp .env.example .env`. `-X utf8` нужен публичным скриптам в Windows. `make` необязателен; `make check`, `make test`, `make student-test`, `make checks` соответствуют прямым командам Python. -## Доказательства критериев +API: `http://127.0.0.1:8080`; HAProxy stats: `http://127.0.0.1:8404`. Если lab4 уже использует эти порты, остановите её стенд либо задайте другие `APP_PUBLISHED_PORT` / `LB_STATS_PORT` и передайте тестам `--base-url`. Проект Compose `mtusi-lab5` имеет собственные volumes и secrets. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +`init_secrets.py` создаёт случайные пароли только при отсутствии файлов. Не удаляйте `secrets/`, сохраняя старые volumes: пароли PostgreSQL не меняются вместе с новым локальным файлом. `.env` и `secrets/` не входят в Git или build context. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Сервис | Роль и версия | Persistence / готовность | +| --- | --- | --- | +| api ×2 | Python 3.14.7, собственный образ lab5 | без постоянного локального состояния; readiness PostgreSQL + session Valkey + S3 | +| lb | HAProxy 3.4.5 | round-robin, readiness API, наружу только 127.0.0.1 | +| postgres | PostgreSQL 18.6 | pgdata, pg_isready; метаданные, jobs, outbox, leases, audit | +| valkey | Valkey 9.0.6 | valkeydata, AOF appendfsync always; только сессии | +| s3 | SeaweedFS 3.85 | s3data, включая filer index; приватный bucket | +| broker | отдельный Valkey 9.0.6 Streams | brokerdata, AOF appendfsync always, noeviction; не Pub/Sub | +| dispatcher | отдельный Python-процесс | восстанавливает и публикует PostgreSQL outbox | +| worker | отдельный Python-процесс | bounded decode/resize; default concurrency=1, prefetch=1 | +| migrate / bucket | одноразовые сервисы | идемпотентные миграции и создание приватного bucket до запуска | +| jobs / cleanup | операторские команды, профиль tools | failed/replay/audit и безопасная очистка | -## Эксперименты и отказы +Все внешние образы закреплены тегом и digest, включая брокер. Worker и dispatcher используют тот же собранный из исходников образ, но отдельные процессы. Критичные зависимости проверяются API; worker/broker не входят в readiness API, потому что durable outbox позволяет принять работу во время их недоступности. Ошибки публикации видны в логах dispatcher, состояние задач — оператору в БД/CLI. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Для учебного S3 master volume size ограничен 100 MiB, volume.max=20: дефолтный рост SeaweedFS по 7 томов должен помещаться в одном узле. Это исправляет обнаруженный на новом volume отказ первого PUT при capacity=5; сетевые таймауты приложения ради этого не увеличивались. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Остановка без потери данных: `docker compose stop` или `docker compose down`. **`docker compose down -v` намеренно удаляет все учебные данные lab5**, включая broker и S3. Не используйте её в проверках отказов. -## Ограничения и следующие шаги +## Что изменилось относительно lab4 -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сохранены пользователи, Bearer-сессии, ACL чатов, идемпотентность сообщений, keyset pagination, две API-реплики, PostgreSQL, Valkey и приватное S3. Публичные tests/ и contracts/ не изменены. Устойчивый исходник теперь сначала записывается в `staging/`, а API возвращает **202 со снимком queued**; клиент опрашивает GET метаданных. Worker публикует оригинал и варианты, затем атомарно переводит attachment в ready. + +```text +HTTP upload -> API -> S3 staging (все bytes) + -> PostgreSQL: attachment queued + job + outbox (один COMMIT) + | + dispatcher -> durable Valkey Stream + | + worker + | + PostgreSQL ready + 3 variant metadata <- конечный S3 +``` + +API проверяет сессию, чат, filename, MIME, bytes и пиксели. До 202 все байты уже приняты, проверены и находятся в устойчивом S3, а задание и намерение публикации — в PostgreSQL. Локальный `/tmp` служит только ограниченным временным буфером. API не делает resize и не исполняет работу worker даже при его остановке. В сообщениях брокера только поля `jobs.schema.json`; bytes, credentials, внутренние URL и клиентские пути не передаются. + +Upload и DB намерение фиксируются через `Repository.save_attachment` / `Jobs.save_upload`. PUT staging выполняется перед INSERT/COMMIT. Неопределённая ошибка между PUT и DB может оставить orphan; старая очистка lab4 дополнена защитой staging и variants. После успешного 202 задача уже восстановима без API-процесса. + +Миграция `0005_async_attachments.sql` добавляет status, error_code, staging, attachment_jobs, attachment_outbox, attachment_variants, job_audit и одноразовые операторские fault hooks. При применении поверх lab4 существующие оригиналы становятся staging для нового pipeline; объект, чей staging key совпадает с final key, не удаляется. + +## Outbox, повторы и подтверждения + +Dispatcher арендует строки outbox, публикует XADD и только после успешного ответа отмечает published. COMMIT upload и INSERT outbox атомарны. Если процесс умер после COMMIT до publish, строка остаётся и публикуется после запуска. Если XADD прошёл, но ответ/mark потерян, допустима повторная доставка с тем же job_id. Это **at least once**, а не exactly once. + +Брокер хранит stream и Pending Entries List в AOF с fsync каждой записи. XREADGROUP отдаёт по одной записи на поток; XACK выполняется после устойчивого ready либо после COMMIT решения о retry/failed. XAUTOCLAIM возвращает pending сообщения умершего consumer. Приложение не использует NOACK или обычный Pub/Sub. Stream автоматически не обрезается, чтобы не удалить ещё не подтверждённую работу; рост истории — эксплуатационное ограничение учебного стенда. + +PostgreSQL — источник истины о логической работе. Claim блокирует job и выдаёт lease_token, увеличивает attempt. Только текущая неистёкшая аренда может зафиксировать результат: token, attempt и pipeline_version проверяются при записи и перед завершением транзакции. После гибели worker processing восстанавливается в queued или failed по исчерпанию бюджета. Новый outbox записывается в той же транзакции, что и retry/recovery. Устаревший worker не может перевести ready обратно в processing. + +Финальные object keys детерминированы: `originals/` и `variants//v1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Уникальность метаданных — `(attachment_id,pipeline_version,name)`. Повтор пишет тот же логический набор, ready-доставка подтверждается без повторной обработки. Доступных pipeline несколько не реализовано: используется только версия 1, без исследования трека 5. + +## Изображения и ограничения + +| Вариант | Правило | +| --- | --- | +| original | исходные bytes и MIME; SHA-256 совпадает с upload | +| thumbnail | JPEG quality85, fit256×256, без увеличения, округление вниз минимум1px | +| crop | JPEG quality85, центральный квадрат128×128; маленький исходник может увеличиваться | +| optimized | WebP quality80/method4, fit1280×1280 без увеличения | + +EXIF orientation применяется до resize/crop. Прозрачность компонуется на белый. Варианты не содержат исходных EXIF/ICC/XMP; оригинал неизменен. WebP не обязан быть меньше любого маленького PNG/JPEG. + +Вход: PNG/JPEG ≤10 MiB и ≤40M pixels. API и worker независимо ограничивают bytes; worker сверяет размер и SHA-256 со staging metadata из БД. Декодирование выполняется отдельным процессом с wall timeout `JOB_TIMEOUT_SECONDS=10`, RLIMIT_AS `IMAGE_MEMORY_MIB=768`, CPU/размером выходных файлов и ограниченным tmpfs128MiB. Worker по умолчанию один поток, контейнер1GiB/1CPU; процесс API512MiB. Увеличивать concurrency без согласованного увеличения памяти/tmpfs нельзя. + +S3 connect/read timeout — 1/3 с; PostgreSQL client I/O timeout — 3 с, отдельный таймаут пула — 3 с. Lease45с больше штатного бюджета ограниченной обработки. Если lease истёк, fenced commit не публикует устаревший результат; следующий запуск повторяет детерминированные эффекты. + +## Retry, failed registry и replay + +`JOB_MAX_ATTEMPTS=5`. Постоянные ошибки (invalid_image, image_too_large, integrity mismatch) завершаются failed сразу. Временные ошибки хранилищ повторяются с exponential backoff + equal jitter: `cap=min(JOB_RETRY_MAX_SECONDS, JOB_RETRY_BASE_SECONDS*2^(run_attempt-1))`, delay в `[cap/2,cap]`. Defaults base1с/max30с. Причина и available_at сохраняются; после исчерпания — `retry_exhausted`. + +Реестр failed находится в PostgreSQL, доступен отдельной операторской командой: + +```powershell +docker compose --profile tools run --rm jobs failed +docker compose --profile tools run --rm jobs show JOB_UUID +docker compose --profile tools run --rm jobs replay ATTACHMENT_UUID --actor Chernikov --reason "Причина исправлена" +``` + +Replay разрешён только для failed при сохранённом staging; сохраняет job_id, сбрасывает ограниченный бюджет run_attempts, монотонно продолжает attempt и пишет audit. Replay уже ready — записанный в audit no-op. Задача не превращается в неконтролируемый дубль. Публичного административного endpoint нет; jobs CLI не раскрывает токены и object keys. + +## Очистка и credentials + +Worker удаляет staging только после COMMIT ready. При сбое DELETE или смерти процесса pending_cleanup остаётся в БД; maintenance повторяет операцию. Ready не зависит от успешности удаления staging. Queued/processing staging всегда защищён от очистки. + +Failed-исходники сохраняются минимум24ч для диагностики/replay. Операторская cleanup при истечении retention сначала фиксирует tombstone под attachment row lock (replay после этого запрещён), затем делает идемпотентный S3 DELETE. Сбой между этими шагами оставляет безопасный повтор удаления. Неучтённые объекты очищаются только после grace≥1ч. Учтённые вложения, ещё не прикреплённые к сообщениям, автоматически не удаляются: такого пользовательского lifecycle в контракте нет. + +```powershell +docker compose --profile tools run --rm cleanup +docker compose --profile tools run --rm cleanup --apply +``` + +Оператор запускает cleanup по расписанию, например раз в час. `STAGING_RETENTION_SECONDS` задаёт retention (default86400), `--failed-retention-seconds` позволяет явно его переопределить; значение меньше24ч запрещено. Автоматическое удаление failed в worker не включено. + +API и worker имеют отдельные S3 identity, bootstrap/cleanup — административные. API не получает credentials брокера. S3/broker/БД и порты API-реплик не опубликованы. В SeaweedFS выбранная bucket-scoped Write включает PUT/DELETE; более узкий запрет DELETE у этой identity не заявляется. Внутренняя Compose-сеть доверенная, удалённый доступ требует TLS. + +## Проверки и доказательства + +Стандартный прогон: `python -X utf8 scripts/verify.py`; сбои: `python -X utf8 scripts/lab5_checks.py` или verify с `--faults`. Проверки сбоев сами меняют состояние сервисов и восстанавливают их в finally; volumes не удаляются, tokens остаются только в памяти. + +| Критерий | Сценарий / файл | +| --- | --- | +| Публичный контракт и 202→ready | tests/smoke.py, evidence/smoke.txt | +| Негативные файлы, ACL, queued сообщения, варианты | student-tests, evidence/tests.txt | +| Изображения: EXIF/alpha/размеры/bytes/лимиты | unit-tests/test_image_process.py, evidence/unit.txt | +| Worker остановлен, рестарт API/broker, та же работа ready | lab5_checks --only durability | +| Crash после COMMIT до publish | lab5_checks --only outbox | +| Недоступность broker при приёме 202 | lab5_checks --only broker_outage | +| Повтор ready-задачи и безопасный replay ready | lab5_checks --only redelivery | +| Crash после S3 до COMMIT/ack | lab5_checks --only crash | +| Зависший processing и lease recovery | lab5_checks --only lease | +| Backoff+jitter, 5 попыток, failed/replay | lab5_checks --only retries permanent | +| Очистка staging после ready, защита pending | lab5_checks --only staging | +| Перезапуск всех хранилищ | tests/resilience.py --mode persistence, evidence/persistence.txt | +| Переключение API | tests/resilience.py --mode failover, evidence/failover.txt | + +Фактический прогон 11.11.2026: Windows, Docker Engine 29.8.1 / Compose 5.5.1, Linux containers. Docker VM: 22 CPU и 16 464 109 568 байт RAM. Лимиты каждого сервиса перечислены в Compose и подтверждены snapshot. + +| Проверка | Наблюдение | Доказательство | +| --- | --- | --- | +| Offline и публичный smoke | PASS, 202 queued → ready; оригинал и JPEG256×160 проверены | evidence/offline.txt, smoke.txt | +| Собственные HTTP-тесты | 98/98 PASS за 38,587 с | evidence/tests.txt | +| Unit pipeline/ack/retry | 13/13 PASS за 1,124 с | evidence/unit.txt | +| Все 10 отказных сценариев | PASS, реальные stop/restart/kill; новые job вручную для восстановления не создавались | evidence/lab5-checks.txt | +| Убит worker после всех 4 S3 PUT до ready/ack | processing, staging сохранён, 3 объекта вариантов ещё без metadata; та же job стала ready на попытке 2 | evidence/lab5-checks.txt, crash | +| Истёкший processing | попытка 2 той же job ready; старый lease не может finish или fail | evidence/lab5-checks.txt, lease | +| Jitter | первые задержки 0,5774478 с и 1,0739815 с; соответствуют интервалам [0,5;1] и [1;2], available_at сохранён | evidence/lab5-checks.txt, retries | +| Исчерпание/replay | run_attempts=5 → retry_exhausted; явный replay → тот же job_id, attempt=6 ready; следующий replay no-op | evidence/lab5-checks.txt | +| Общий restart | старый токен, сообщение, metadata и original сохранены | evidence/persistence.txt | +| Failover API | 2 реплики до отказа, 119 успешных записей, 0 ошибок, восстановление записи через 4,5 с | evidence/failover.txt | +| Приостановка PostgreSQL | прямой readiness ограничен клиентским таймаутом, query/COMMIT/ROLLBACK закрывают соединение, тот же пул восстанавливается | evidence/postgres-pause.txt | + +Измерения — короткие функциональные прогоны, не исследование производительности. Нагрузочного сравнения concurrency/prefetch и throughput нет, трек5 не заявлен. Проверка удаления failed staging после реальных24ч не ожидалась сутки; алгоритм проверен по блокировкам/tombstone, а автоматическая очистка ready staging и защита queued подтверждены отдельным сценарием. Полный список реальных состояний контейнеров, закреплённых образов, лимитов и volumes — в `evidence/stack.json`. + +## Ограничения + +Один хост, по одному PostgreSQL, S3 и брокеру: отказ узла/диска не выдерживается. Сигналы/API-process restart выдерживаются при сохранённых volumes. Выбранная очередь не ограничивает admission количеством pending job; backpressure и сравнение настроек concurrency относятся к незаявленному треку5. Нет exactly-once доставки, распределённых транзакций S3/БД/брокера и нескольких обновляемых pipeline. При неопределённом COMMIT сообщения повторяются с тем же Idempotency-Key; upload сам по себе не идемпотентен. + +## Источники и использование ИИ + +- README, COURSE, contracts и tests текущего skeleton; собственная реализация lab4. +- [Valkey Streams](https://valkey.io/topics/streams-intro/), [XREADGROUP](https://valkey.io/commands/xreadgroup/), [XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), [XACK](https://valkey.io/commands/xack/). +- [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [Image file formats](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html). +- [psycopg 3.3.6](https://github.com/psycopg/psycopg/tree/3.3.6), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). +- Codex (OpenAI): помощь в реализации, проверках и подготовке отчёта. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3f05149 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,295 @@ +# Стенд lab5: API -> S3 staging + PostgreSQL outbox -> Valkey Streams -> worker. +# Перед первым запуском: python scripts/init_secrets.py +# Запуск: docker compose up -d --build --wait (подробно — REPORT.md, раздел «Запуск из чистого clone»). +name: mtusi-lab5 + +x-app: &app + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-lab5} + image: mtusi-messenger:lab5 + # Имя остановленного контейнера встроенный DNS Docker пересылает внешнему серверу и ждёт ответа + # около 4 с (замерено в lab2). Ограничиваем ожидание DNS одной секундой. + dns_opt: + - timeout:1 + - attempts:1 + +x-api-environment: &api-environment + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + APP_LOG_LEVEL: ${APP_LOG_LEVEL:-INFO} + APP_SHUTDOWN_TIMEOUT: ${APP_SHUTDOWN_TIMEOUT:-8} + APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-3} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_SIZE: ${DB_POOL_SIZE:-10} + # Пароли передаются файлами секретов, а не переменными окружения. + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + +x-job-environment: &job-environment + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-10} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-45} + IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:-768} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-1} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-30} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + +services: + # Единственная публичная точка входа. Проверяет /health/ready каждой реплики каждую секунду. + lb: + image: haproxy:3.4.5-trixie@sha256:76928c0d6b39bdd5f1c15d519cf48c47a6aa18c1dc552f7af1c146d2aa003c14 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PUBLISHED_PORT:-8080}:8080" + # Страница состояния HAProxy (UP/DOWN реплик), только для этой машины. + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + restart: unless-stopped + + # Реплики API: без container_name и без опубликованных портов, доступ только через lb. + api: + <<: *app + environment: *api-environment + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + tmpfs: + - /tmp:size=32M,mode=1777 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.50", memory: 512M} + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + bucket: + condition: service_completed_successfully + stop_grace_period: 10s + restart: unless-stopped + + # Одноразовый сервис: применяет миграции один раз до старта реплик и завершается. + migrate: + <<: *app + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + restart: "no" + + # S3 gateway, filer, master и volume в одном учебном контейнере; все данные в volume. + # Ни S3, ни административные HTTP-порты на хост не публикуются. + s3: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + command: ["server", "-ip=s3", "-dir=/data", "-volume.max=20", "-master.volumeSizeLimitMB=100", + "-filer", "-filer.port=8888", "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: + - s3data:/data + - ./infra/seaweedfs/filer.toml:/etc/seaweedfs/filer.toml:ro + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + restart: unless-stopped + + bucket: + <<: *app + command: ["python", "-m", "messenger.bucket"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [s3_admin_access_key_id, s3_admin_secret_access_key] + depends_on: [s3] + restart: "no" + + # Только явный запуск оператором, без административного ключа у API. + cleanup: + <<: *app + profiles: [tools] + entrypoint: ["python", "-m", "messenger.cleanup_objects"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:8333 + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: [database_url, s3_admin_access_key_id, s3_admin_secret_access_key] + depends_on: + postgres: {condition: service_healthy} + bucket: {condition: service_completed_successfully} + restart: "no" + + broker: + image: valkey/valkey:9.0.6-trixie@sha256:63346cb24a61221e76bdf41acce99b3968a9fa83d8122144deab45394b27b4f2 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + secrets: [broker_auth] + volumes: + - brokerdata:/data + - ./infra/broker/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_auth)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.50", memory: 256M} + restart: unless-stopped + + dispatcher: + <<: *app + command: ["python", "-m", "messenger.dispatcher"] + environment: *job-environment + secrets: [database_url, broker_url, s3_worker_access_key_id, s3_worker_secret_access_key] + # Readiness API здесь не подходит: dispatcher сам переживает отказ брокера. + healthcheck: {disable: true} + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + bucket: {condition: service_completed_successfully} + deploy: + resources: + limits: {cpus: "0.50", memory: 256M} + stop_grace_period: 8s + restart: unless-stopped + + worker: + <<: *app + command: ["python", "-m", "messenger.worker"] + environment: *job-environment + secrets: [database_url, broker_url, s3_worker_access_key_id, s3_worker_secret_access_key] + tmpfs: + - /tmp:size=128M,mode=1777 + healthcheck: {disable: true} + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + bucket: {condition: service_completed_successfully} + deploy: + resources: + limits: {cpus: "1.00", memory: 1G} + stop_grace_period: 30s + restart: unless-stopped + + jobs: + <<: *app + profiles: [tools] + entrypoint: ["python", "-m", "messenger.jobs_cli"] + environment: *job-environment + secrets: [database_url, broker_url, s3_worker_access_key_id, s3_worker_secret_access_key] + healthcheck: {disable: true} + depends_on: + migrate: {condition: service_completed_successfully} + restart: "no" + + postgres: + image: postgres:18.6-trixie@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + # С PostgreSQL 18 образ хранит данные в /var/lib/postgresql/18/docker: volume монтируется на уровень выше. + - pgdata:/var/lib/postgresql + healthcheck: + test: ["CMD", "pg_isready", "--username=messenger", "--dbname=messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + restart: unless-stopped + + valkey: + image: valkey/valkey:9.0.6-trixie@sha256:63346cb24a61221e76bdf41acce99b3968a9fa83d8122144deab45394b27b4f2 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + secrets: [valkey_auth] + volumes: + - valkeydata:/data + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + healthcheck: + # Пароль берётся из того же файла секрета, что подключает valkey.conf. + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_auth)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.50", memory: 256M} + restart: unless-stopped + +volumes: + pgdata: + valkeydata: + s3data: + brokerdata: + +# Файлы создаёт scripts/init_secrets.py; каталог secrets/ в .gitignore. +# Каждому сервису выдаются только нужные ему секреты; lb не получает ни одного. +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + valkey_auth: + file: ./secrets/valkey_auth + cache_url: + file: ./secrets/cache_url + broker_auth: + file: ./secrets/broker_auth + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_worker_access_key_id: + file: ./secrets/s3_worker_access_key_id + s3_worker_secret_access_key: + file: ./secrets/s3_worker_secret_access_key diff --git a/evidence/README.md b/evidence/README.md index 8df7001..d589375 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,21 @@ # Доказательства +Фактическая проверка lab5 на4 выполнена11.11.2026 в локальном Docker Desktop. + +- `offline.txt` — структура skeleton, refs и Python syntax, PASS. +- `unit.txt` — 13 тестов декодера и worker ack/retry, PASS. +- `smoke.txt` — публичный сценарий lab5, PASS. +- `tests.txt` — 98 собственных HTTP-тестов, PASS. +- `lab5-checks.txt` — все10 сценариев grade3/4 (durability, outbox, broker outage, redelivery, crash, lease, retry/replay, permanent failure, staging). +- `persistence.txt`, `failover.txt` — публичные проверки restart/failover с реальными хуками. +- `postgres-pause.txt` — сохранённая регрессия клиентского таймаута PostgreSQL из lab4. +- `cleanup.txt` — dry-run orphan/failed staging cleanup на новой схеме lab5, без удаления данных. +- `stack.json` — фактические процессы, времена старта, версии, лимиты, volumes и порты; переменные окружения/секреты не сохраняются. + +Повторить стандартные проверки: `python -X utf8 scripts/verify.py`. +Полный прогон с управляемыми отказами: `python -X utf8 scripts/verify.py --faults`. +Отказы нельзя запускать параллельно друг с другом. Скрипты восстанавливают сервисы и сохраняют volumes. Методика и ограничения — в REPORT.md. + Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. diff --git a/evidence/cleanup.txt b/evidence/cleanup.txt new file mode 100644 index 0000000..a743349 --- /dev/null +++ b/evidence/cleanup.txt @@ -0,0 +1,3 @@ +{"scanned": 77, "fresh": 77, "referenced": 0, "orphan": 0, "deleted": 0, "apply": false, "seconds": 0.093} + Container mtusi-lab5-cleanup-run-f64711ee55d4 Creating + Container mtusi-lab5-cleanup-run-f64711ee55d4 Created diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..c505e1f --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 4.5} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab5-checks.txt b/evidence/lab5-checks.txt new file mode 100644 index 0000000..a689276 --- /dev/null +++ b/evidence/lab5-checks.txt @@ -0,0 +1,23 @@ +START stack 2026-11-11T20:32:07.954371+00:00 +PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory +START durability 2026-11-11T20:32:08.262127+00:00 +PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=1d457f73-a915-41c9-913c-5a0014df0511 +START outbox 2026-11-11T20:32:38.208630+00:00 +PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=6b472c6b-52ec-432f-8472-b27fb56598c1 +START broker_outage 2026-11-11T20:33:07.567493+00:00 +PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=a363aa46-6f18-45cb-9fe2-109f83086db7 +START redelivery 2026-11-11T20:33:25.177987+00:00 +PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=f951285c-bd7a-4763-b3c5-3a6b649d9a22 +START crash 2026-11-11T20:33:34.557181+00:00 +PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=ea420ed9-8c7c-47e7-9d58-a5ccc923b782 attempts=2 window={"final": true, "staging": true, "staging_deleted": false, "status": "processing", "variant_objects": 3} +START lease 2026-11-11T20:34:42.684279+00:00 +PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=766e3884-a0df-426c-94c5-f1ba947b6e2a old_lease_until=2026-11-11T20:35:39.564619+00:00 attempts=2 +PASS fencing: old lease cannot finish or fail after recovered ready commit +START retries 2026-11-11T20:35:52.377383+00:00 +PASS transient errors retry with durable backoff/jitter; job_id=cb2a9208-6580-4003-a015-93873c7191bc audit=[{"action": "retry", "actor": "f7611f3a576d-fb830a49-0", "created_at": "2026-11-11T20:36:05.032253+00:00", "details": {"attempt": 1, "available_at": "2026-11-11T20:36:05.607447Z", "delay_seconds": 0.5774477733124553, "run_attempts": 1}, "reason": "injected_transient"}, {"action": "retry", "actor": "f7611f3a576d-fb830a49-0", "created_at": "2026-11-11T20:36:05.777829+00:00", "details": {"attempt": 2, "available_at": "2026-11-11T20:36:06.849152Z", "delay_seconds": 1.0739815038117153, "run_attempts": 2}, "reason": "injected_transient"}] +PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=6a9cc04b-cc23-4e90-9788-f185773a53be max_attempts=5 error_code=retry_exhausted replay={"attachment_id": "0640b252-52f0-42d6-af9d-6a9bc1743b64", "attempt": 6, "error_code": null, "job_id": "6a9cc04b-cc23-4e90-9788-f185773a53be", "run_attempts": 1, "status": "ready"} +START permanent 2026-11-11T20:37:01.785708+00:00 +PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=c4d6f975-751a-4208-9e49-e9e34c15f9d2 error_code=invalid_image +START staging 2026-11-11T20:37:19.184496+00:00 +PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=6425f731-feab-466c-bfa1-31596c1cacb3 +PASS selected lab5 grade 3/4 checks; no grade 5 track claimed diff --git a/evidence/offline.txt b/evidence/offline.txt new file mode 100644 index 0000000..af45ae2 --- /dev/null +++ b/evidence/offline.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..37795bf --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/postgres-pause.txt b/evidence/postgres-pause.txt new file mode 100644 index 0000000..61ecb94 --- /dev/null +++ b/evidence/postgres-pause.txt @@ -0,0 +1,10 @@ +{"cycle": 1, "api": 1, "status": 503, "requests": 12, "seconds": [3.019, 3.079, 3.075, 3.073, 3.066, 3.064, 3.064, 3.065, 3.01, 3.009, 3.011, 3.012], "live_seconds": 0.012} +{"cycle": 1, "api": 2, "status": 503, "requests": 12, "seconds": [3.026, 3.009, 3.013, 3.007, 3.01, 3.011, 3.013, 3.012, 3.012, 3.014, 3.012, 3.014], "live_seconds": 0.005} +{"cycle": 1, "recovered_seconds": 0.677, "existing_session_and_write": "PASS"} +{"cycle": 2, "api": 1, "status": 503, "requests": 12, "seconds": [3.09, 3.089, 3.087, 3.088, 3.069, 3.065, 3.009, 3.007, 3.007, 3.008, 3.007, 3.007], "live_seconds": 0.005} +{"cycle": 2, "api": 2, "status": 503, "requests": 12, "seconds": [3.017, 3.015, 3.007, 3.005, 3.006, 3.008, 3.009, 3.006, 3.007, 3.008, 3.007, 3.007], "live_seconds": 0.006} +{"cycle": 2, "recovered_seconds": 0.897, "existing_session_and_write": "PASS"} +{"phase": "query", "timeout_seconds": 3.002, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +{"phase": "commit", "timeout_seconds": 3.001, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +{"phase": "rollback", "timeout_seconds": 3.002, "connection_closed": true, "pool_max_size": 1, "recovered_seconds": 0.002} +PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..be75495 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.json b/evidence/stack.json new file mode 100644 index 0000000..bb72140 --- /dev/null +++ b/evidence/stack.json @@ -0,0 +1,182 @@ +{ + "captured_at": "2026-11-11T20:48:41.179173+00:00", + "containers": [ + { + "name": "mtusi-lab5-api-1", + "image": "mtusi-messenger:lab5", + "status": "running", + "started_at": "2026-11-11T20:46:17.2642711Z", + "health": "healthy", + "memory_limit_bytes": 536870912, + "cpu_limit_nanos": 500000000, + "volumes": [], + "published_ports": { + "8080/tcp": null + } + }, + { + "name": "mtusi-lab5-api-2", + "image": "mtusi-messenger:lab5", + "status": "running", + "started_at": "2026-11-11T20:45:47.113222398Z", + "health": "healthy", + "memory_limit_bytes": 536870912, + "cpu_limit_nanos": 500000000, + "volumes": [], + "published_ports": { + "8080/tcp": null + } + }, + { + "name": "mtusi-lab5-broker-1", + "image": "valkey/valkey:9.0.6-trixie@sha256:63346cb24a61221e76bdf41acce99b3968a9fa83d8122144deab45394b27b4f2", + "status": "running", + "started_at": "2026-11-11T20:45:26.310735977Z", + "health": "healthy", + "memory_limit_bytes": 268435456, + "cpu_limit_nanos": 500000000, + "volumes": [ + { + "name": "mtusi-lab5_brokerdata", + "destination": "/data" + } + ], + "published_ports": { + "6379/tcp": null + } + }, + { + "name": "mtusi-lab5-bucket-1", + "image": "mtusi-messenger:lab5", + "status": "exited", + "started_at": "2026-11-11T20:47:01.904404105Z", + "health": "unhealthy", + "memory_limit_bytes": 0, + "cpu_limit_nanos": 0, + "volumes": [], + "published_ports": {} + }, + { + "name": "mtusi-lab5-dispatcher-1", + "image": "mtusi-messenger:lab5", + "status": "running", + "started_at": "2026-11-11T20:45:46.648528264Z", + "health": null, + "memory_limit_bytes": 268435456, + "cpu_limit_nanos": 500000000, + "volumes": [], + "published_ports": { + "8080/tcp": null + } + }, + { + "name": "mtusi-lab5-lb-1", + "image": "haproxy:3.4.5-trixie@sha256:76928c0d6b39bdd5f1c15d519cf48c47a6aa18c1dc552f7af1c146d2aa003c14", + "status": "running", + "started_at": "2026-11-11T20:45:50.374957227Z", + "health": null, + "memory_limit_bytes": 134217728, + "cpu_limit_nanos": 500000000, + "volumes": [], + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ], + "8404/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8404" + } + ] + } + }, + { + "name": "mtusi-lab5-migrate-1", + "image": "mtusi-messenger:lab5", + "status": "exited", + "started_at": "2026-11-11T20:46:13.806559806Z", + "health": "unhealthy", + "memory_limit_bytes": 134217728, + "cpu_limit_nanos": 500000000, + "volumes": [], + "published_ports": {} + }, + { + "name": "mtusi-lab5-postgres-1", + "image": "postgres:18.6-trixie@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336", + "status": "running", + "started_at": "2026-11-11T20:45:26.382080938Z", + "health": "healthy", + "memory_limit_bytes": 536870912, + "cpu_limit_nanos": 1000000000, + "volumes": [ + { + "name": "mtusi-lab5_pgdata", + "destination": "/var/lib/postgresql" + } + ], + "published_ports": { + "5432/tcp": null + } + }, + { + "name": "mtusi-lab5-s3-1", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "status": "running", + "started_at": "2026-11-11T20:45:29.272738761Z", + "health": null, + "memory_limit_bytes": 536870912, + "cpu_limit_nanos": 1000000000, + "volumes": [ + { + "name": "mtusi-lab5_s3data", + "destination": "/data" + } + ], + "published_ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + } + }, + { + "name": "mtusi-lab5-valkey-1", + "image": "valkey/valkey:9.0.6-trixie@sha256:63346cb24a61221e76bdf41acce99b3968a9fa83d8122144deab45394b27b4f2", + "status": "running", + "started_at": "2026-11-11T20:45:26.514802669Z", + "health": "healthy", + "memory_limit_bytes": 268435456, + "cpu_limit_nanos": 500000000, + "volumes": [ + { + "name": "mtusi-lab5_valkeydata", + "destination": "/data" + } + ], + "published_ports": { + "6379/tcp": null + } + }, + { + "name": "mtusi-lab5-worker-1", + "image": "mtusi-messenger:lab5", + "status": "running", + "started_at": "2026-11-11T20:45:46.903676916Z", + "health": null, + "memory_limit_bytes": 1073741824, + "cpu_limit_nanos": 1000000000, + "volumes": [], + "published_ports": { + "8080/tcp": null + } + } + ] +} diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..07e61f9 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1 @@ +Saved evidence/stack.json (process start times, versions, limits, volumes and ports) diff --git a/evidence/tests.txt b/evidence/tests.txt new file mode 100644 index 0000000..3476eb8 --- /dev/null +++ b/evidence/tests.txt @@ -0,0 +1,103 @@ +test_authentication_is_checked_before_body_and_resource (test_access.AuthenticationOrderTest.test_authentication_is_checked_before_body_and_resource) ... ok +test_registration_does_not_need_a_token (test_access.AuthenticationOrderTest.test_registration_does_not_need_a_token) ... ok +test_chat_list_contains_only_own_chats (test_access.ForeignChatTest.test_chat_list_contains_only_own_chats) ... ok +test_every_member_can_read_and_write (test_access.ForeignChatTest.test_every_member_can_read_and_write) ... ok +test_foreign_chat_looks_like_missing_chat (test_access.ForeignChatTest.test_foreign_chat_looks_like_missing_chat) ... ok +test_invalid_chat_id_is_400 (test_access.ForeignChatTest.test_invalid_chat_id_is_400) ... ok +test_outsider_cannot_get_chat (test_access.ForeignChatTest.test_outsider_cannot_get_chat) ... ok +test_outsider_cannot_read_history (test_access.ForeignChatTest.test_outsider_cannot_read_history) ... ok +test_outsider_cannot_write (test_access.ForeignChatTest.test_outsider_cannot_write) ... ok +test_unknown_route_is_404 (test_access.ForeignChatTest.test_unknown_route_is_404) ... ok +test_x_user_id_does_not_open_a_foreign_chat (test_access.ForeignChatTest.test_x_user_id_does_not_open_a_foreign_chat) ... ok +test_attachment_only_and_idempotency (test_attachments.AttachmentsTest.test_attachment_only_and_idempotency) ... ok +test_byte_limit_before_body (test_attachments.AttachmentsTest.test_byte_limit_before_body) ... ok +test_corrupt_and_wrong_mime (test_attachments.AttachmentsTest.test_corrupt_and_wrong_mime) ... ok +test_exif_orientation_applies_to_variants_and_original_stays_identical (test_attachments.AttachmentsTest.test_exif_orientation_applies_to_variants_and_original_stays_identical) ... ok +test_filename (test_attachments.AttachmentsTest.test_filename) ... ok +test_foreign_chat_upload (test_attachments.AttachmentsTest.test_foreign_chat_upload) ... ok +test_jpeg_original (test_attachments.AttachmentsTest.test_jpeg_original) ... ok +test_large_optimized_image_is_bounded (test_attachments.AttachmentsTest.test_large_optimized_image_is_bounded) ... ok +test_original_and_metadata_and_acl (test_attachments.AttachmentsTest.test_original_and_metadata_and_acl) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentsTest.test_pixel_limit_before_decode) ... ok +test_small_transparent_image_does_not_upscale_thumbnail_or_optimized (test_attachments.AttachmentsTest.test_small_transparent_image_does_not_upscale_thumbnail_or_optimized) ... ok +test_unknown_foreign_and_duplicate_ids (test_attachments.AttachmentsTest.test_unknown_foreign_and_duplicate_ids) ... ok +test_errors_also_carry_instance_id (test_balancing.BalancingTest.test_errors_also_carry_instance_id) ... ok +test_message_written_on_one_replica_is_read_on_another (test_balancing.BalancingTest.test_message_written_on_one_replica_is_read_on_another) ... ok +test_one_session_is_served_by_several_replicas (test_balancing.BalancingTest.test_one_session_is_served_by_several_replicas) ... ok +test_100_messages_from_10_clients (test_concurrency.ConcurrencyTest.test_100_messages_from_10_clients) ... ok +test_chats_created_in_parallel_are_all_listed (test_concurrency.ConcurrencyTest.test_chats_created_in_parallel_are_all_listed) ... ok +test_same_username_registered_in_parallel_wins_once (test_concurrency.ConcurrencyTest.test_same_username_registered_in_parallel_wins_once) ... ok +test_failed_validation_is_not_remembered (test_idempotency.IdempotencyTest.test_failed_validation_is_not_remembered) ... ok +test_invalid_keys_are_400 (test_idempotency.IdempotencyTest.test_invalid_keys_are_400) ... ok +test_json_formatting_does_not_matter (test_idempotency.IdempotencyTest.test_json_formatting_does_not_matter) ... ok +test_key_format_limits_are_accepted (test_idempotency.IdempotencyTest.test_key_format_limits_are_accepted) ... ok +test_key_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_key_scope_is_user_and_chat) ... ok +test_missing_key_is_400 (test_idempotency.IdempotencyTest.test_missing_key_is_400) ... ok +test_parallel_requests_with_one_key_create_one_message (test_idempotency.IdempotencyTest.test_parallel_requests_with_one_key_create_one_message) ... ok +test_replay_checks_the_session_again (test_idempotency.IdempotencyTest.test_replay_checks_the_session_again) ... ok +test_replay_returns_the_same_message (test_idempotency.IdempotencyTest.test_replay_returns_the_same_message) ... ok +test_same_key_with_another_body_is_409 (test_idempotency.IdempotencyTest.test_same_key_with_another_body_is_409) ... ok +test_chats_are_paged_in_creation_order (test_pagination.ChatListPaginationTest.test_chats_are_paged_in_creation_order) ... ok +test_cursor_of_another_user_is_400 (test_pagination.ChatListPaginationTest.test_cursor_of_another_user_is_400) ... ok +test_default_limit_is_50 (test_pagination.DefaultLimitTest.test_default_limit_is_50) ... ok +test_empty_history (test_pagination.GrowingHistoryTest.test_empty_history) ... ok +test_messages_sent_during_walk_are_not_lost (test_pagination.GrowingHistoryTest.test_messages_sent_during_walk_are_not_lost) ... ok +test_cursor_from_another_chat_is_400 (test_pagination.MessagePaginationTest.test_cursor_from_another_chat_is_400) ... ok +test_default_and_maximum_limit (test_pagination.MessagePaginationTest.test_default_and_maximum_limit) ... ok +test_invalid_cursor_is_400 (test_pagination.MessagePaginationTest.test_invalid_cursor_is_400) ... ok +test_invalid_limit_is_400 (test_pagination.MessagePaginationTest.test_invalid_limit_is_400) ... ok +test_last_full_page_has_no_cursor (test_pagination.MessagePaginationTest.test_last_full_page_has_no_cursor) ... ok +test_limit_can_change_between_pages (test_pagination.MessagePaginationTest.test_limit_can_change_between_pages) ... ok +test_one_message_per_page (test_pagination.MessagePaginationTest.test_one_message_per_page) ... ok +test_walk_over_several_pages (test_pagination.MessagePaginationTest.test_walk_over_several_pages) ... ok +test_basic_is_rejected_on_business_endpoints (test_sessions.BearerTest.test_basic_is_rejected_on_business_endpoints) ... ok +test_protected_endpoints_need_a_token (test_sessions.BearerTest.test_protected_endpoints_need_a_token) ... ok +test_scheme_name_is_case_insensitive (test_sessions.BearerTest.test_scheme_name_is_case_insensitive) ... ok +test_token_selects_the_user (test_sessions.BearerTest.test_token_selects_the_user) ... ok +test_unknown_or_malformed_token (test_sessions.BearerTest.test_unknown_or_malformed_token) ... ok +test_x_user_id_is_ignored (test_sessions.BearerTest.test_x_user_id_is_ignored) ... ok +test_credentials_in_json_body_are_not_accepted (test_sessions.LoginTest.test_credentials_in_json_body_are_not_accepted) ... ok +test_each_login_gives_a_new_token (test_sessions.LoginTest.test_each_login_gives_a_new_token) ... ok +test_malformed_basic_header (test_sessions.LoginTest.test_malformed_basic_header) ... ok +test_password_with_colons_and_unicode (test_sessions.LoginTest.test_password_with_colons_and_unicode) ... ok +test_registration_never_returns_password (test_sessions.LoginTest.test_registration_never_returns_password) ... ok +test_successful_login (test_sessions.LoginTest.test_successful_login) ... ok +test_unknown_user_gets_the_same_answer (test_sessions.LoginTest.test_unknown_user_gets_the_same_answer) ... ok +test_wrong_password (test_sessions.LoginTest.test_wrong_password) ... ok +test_login_again_after_logout (test_sessions.LogoutTest.test_login_again_after_logout) ... ok +test_logout_revokes_only_the_current_token (test_sessions.LogoutTest.test_logout_revokes_only_the_current_token) ... ok +test_bad_member_ids (test_validation.ChatValidationTest.test_bad_member_ids) ... ok +test_bad_titles (test_validation.ChatValidationTest.test_bad_titles) ... ok +test_caller_must_be_a_member (test_validation.ChatValidationTest.test_caller_must_be_a_member) ... ok +test_duplicate_members (test_validation.ChatValidationTest.test_duplicate_members) ... ok +test_unknown_member_rolls_back_the_whole_chat (test_validation.ChatValidationTest.test_unknown_member_rolls_back_the_whole_chat) ... ok +test_valid_chat_keeps_member_order (test_validation.ChatValidationTest.test_valid_chat_keeps_member_order) ... ok +test_extra_fields (test_validation.InvalidJsonTest.test_extra_fields) ... ok +test_invalid_utf8 (test_validation.InvalidJsonTest.test_invalid_utf8) ... ok +test_json_that_is_not_an_object (test_validation.InvalidJsonTest.test_json_that_is_not_an_object) ... ok +test_lone_surrogate_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_lone_surrogate_is_rejected_not_crashed) ... ok +test_malformed_json_on_message (test_validation.InvalidJsonTest.test_malformed_json_on_message) ... ok +test_malformed_json_on_register (test_validation.InvalidJsonTest.test_malformed_json_on_register) ... ok +test_nul_character_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_nul_character_is_rejected_not_crashed) ... ok +test_wrong_content_type (test_validation.InvalidJsonTest.test_wrong_content_type) ... ok +test_wrong_types (test_validation.InvalidJsonTest.test_wrong_types) ... ok +test_empty_message_is_400 (test_validation.MessageValidationTest.test_empty_message_is_400) ... ok +test_missing_or_wrong_text (test_validation.MessageValidationTest.test_missing_or_wrong_text) ... ok +test_nul_in_text_is_400 (test_validation.MessageValidationTest.test_nul_in_text_is_400) ... ok +test_oversized_body_is_413 (test_validation.MessageValidationTest.test_oversized_body_is_413) ... ok +test_sender_id_in_body_is_rejected (test_validation.MessageValidationTest.test_sender_id_in_body_is_rejected) ... ok +test_text_length_limit (test_validation.MessageValidationTest.test_text_length_limit) ... ok +test_unicode_text_is_kept (test_validation.MessageValidationTest.test_unicode_text_is_kept) ... ok +test_bad_display_names (test_validation.UserValidationTest.test_bad_display_names) ... ok +test_bad_passwords (test_validation.UserValidationTest.test_bad_passwords) ... ok +test_bad_usernames (test_validation.UserValidationTest.test_bad_usernames) ... ok +test_duplicate_username_is_409 (test_validation.UserValidationTest.test_duplicate_username_is_409) ... ok +test_missing_fields (test_validation.UserValidationTest.test_missing_fields) ... ok +test_password_length_limits_are_accepted (test_validation.UserValidationTest.test_password_length_limits_are_accepted) ... ok +test_unicode_display_name_is_kept (test_validation.UserValidationTest.test_unicode_display_name_is_kept) ... ok +test_username_length_limits_are_accepted (test_validation.UserValidationTest.test_username_length_limits_are_accepted) ... ok + +---------------------------------------------------------------------- +Ran 98 tests in 38.587s + +OK diff --git a/evidence/unit.txt b/evidence/unit.txt new file mode 100644 index 0000000..bc34486 --- /dev/null +++ b/evidence/unit.txt @@ -0,0 +1,23 @@ + Container mtusi-lab5-worker-run-cfdd5f1b2acf Creating + Container mtusi-lab5-worker-run-cfdd5f1b2acf Created +test_all_variants_and_original_checksum (test_image_process.ImagePipelineTests.test_all_variants_and_original_checksum) ... ok +test_center_crop_uses_center_pixels (test_image_process.ImagePipelineTests.test_center_crop_uses_center_pixels) ... ok +test_decoder_subprocess_under_production_limits (test_image_process.ImagePipelineTests.test_decoder_subprocess_under_production_limits) ... ok +test_exif_orientation_applied_and_metadata_removed (test_image_process.ImagePipelineTests.test_exif_orientation_applied_and_metadata_removed) ... ok +test_floor_dimensions_and_extreme_aspect_ratio (test_image_process.ImagePipelineTests.test_floor_dimensions_and_extreme_aspect_ratio) ... ok +test_invalid_pixels_and_mime_rejected (test_image_process.ImagePipelineTests.test_invalid_pixels_and_mime_rejected) ... ok +test_small_alpha_is_flattened_white_without_upscale (test_image_process.ImagePipelineTests.test_small_alpha_is_flattened_white_without_upscale) ... ok +test_ack_follows_committed_result (test_image_process.WorkerDurabilityTests.test_ack_follows_committed_result) ... ok +test_active_redelivery_does_not_ack_and_ready_is_noop (test_image_process.WorkerDurabilityTests.test_active_redelivery_does_not_ack_and_ready_is_noop) ... ok +test_backoff_bounds_and_cap (test_image_process.WorkerDurabilityTests.test_backoff_bounds_and_cap) ... ok +test_lost_fence_is_left_pending (test_image_process.WorkerDurabilityTests.test_lost_fence_is_left_pending) ... ok +test_permanent_failure_has_no_automatic_processing_retry (test_image_process.WorkerDurabilityTests.test_permanent_failure_has_no_automatic_processing_retry) ... event=job_error job_id=019967da-2cfb-4b85-b14a-b1f8fad53f1e attempt=1 code=invalid_image permanent=True retry_delay=0.886 error_type=ProcessingError +ok +test_retry_ack_requires_durable_failure_record (test_image_process.WorkerDurabilityTests.test_retry_ack_requires_durable_failure_record) ... event=job_error job_id=8a03b517-904d-42c5-b874-3d514a22c796 attempt=1 code=dependency_unavailable permanent=False retry_delay=0.552 error_type=ProcessingError +event=job_error job_id=8a03b517-904d-42c5-b874-3d514a22c796 attempt=1 code=dependency_unavailable permanent=False retry_delay=0.877 error_type=ProcessingError +ok + +---------------------------------------------------------------------- +Ran 13 tests in 1.124s + +OK diff --git a/infra/README.md b/infra/README.md index 5a39e86..a2c8c73 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,5 +1,16 @@ # Инфраструктура студента +Реализация находится в корневом `compose.yaml` проекта `mtusi-lab5`: + +- HAProxy → две API-реплики → PostgreSQL + session Valkey + приватный SeaweedFS S3. +- Отдельный брокер Valkey Streams: `broker/valkey.conf`, AOF appendfsync always, volume brokerdata, noeviction. +- Отдельные dispatcher (PostgreSQL outbox) и worker (decode/resize, bounded concurrency/prefetch). +- `seaweedfs/filer.toml` хранит индекс filer в persistent volume; bucket bootstrap идемпотентен. +- secrets создаются `scripts/init_secrets.py`; у API/worker отдельные S3 credentials, административный ключ только у bootstrap/cleanup. +- операторские CLI находятся в профиле tools: `jobs failed/show/replay` и `cleanup`. + +Версии/digest, готовность, лимиты и воспроизведение отказов подробно описаны в REPORT.md и DEMO.md. Публичные внутренние порты не открываются в обход LB. + Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. diff --git a/infra/broker/valkey.conf b/infra/broker/valkey.conf new file mode 100644 index 0000000..78b6f9a --- /dev/null +++ b/infra/broker/valkey.conf @@ -0,0 +1,11 @@ +# Отдельный Valkey Streams, не session store. AOF fsync до ответа на запись. +include /run/secrets/broker_auth +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data +appendonly yes +appendfsync always +save "" +maxmemory 128mb +maxmemory-policy noeviction diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..116a007 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,72 @@ +# HAProxy для lab3: единая точка входа, round-robin между репликами API. +# Реплики находятся через DNS Docker и исключаются по активной проверке /health/ready. + +global + log stdout format raw local0 info + maxconn 4096 + +# Встроенный DNS Docker: имя сервиса api возвращает адреса всех запущенных реплик. +# Список перечитывается каждую секунду, поэтому новые и удалённые реплики видны сразу. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + # Соединение к реплике закрывается после каждого ответа: запрос не уйдёт по старому + # соединению в реплику, которая уже закрывает приём. + option http-server-close + timeout connect 1s + timeout client 30s + timeout server 15s + timeout http-request 10s + timeout http-keep-alive 10s + timeout check 1s + # Повтор только если соединение с репликой не установилось: запрос до неё не дошёл, + # поэтому повтор на другой реплике безопасен даже для POST. + retries 2 + retry-on conn-failure + option redispatch 1 + # Ошибки самого балансировщика — в формате ошибок контракта (без X-Instance-Id: его ставит только API). + unique-id-format %[uuid()] + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"API replica failed to respond","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + # Реплика готова, только пока /health/ready отвечает 200. После SIGTERM реплика сама + # начинает отвечать 503 и ещё 3 с обслуживает запросы (APP_DRAIN_DELAY). + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + # inter 1s: проверка раз в секунду; после первой неудачи — раз в 0,5 с (fastinter); + # fall 2: реплика исключается после двух неудач подряд, то есть примерно за 1,5–2,5 с. + # rise 2: возвращается после двух успешных проверок. + default-server check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 1-8 api:8080 + +# Страница состояния: какие реплики UP/DOWN. Опубликована только на 127.0.0.1. +frontend stats + bind :8404 + stats enable + stats uri / + stats refresh 2s + # Показывать адреса реплик: по ним видно, какой контейнер в каком состоянии. + stats show-legends diff --git a/infra/seaweedfs/filer.toml b/infra/seaweedfs/filer.toml new file mode 100644 index 0000000..b63f04d --- /dev/null +++ b/infra/seaweedfs/filer.toml @@ -0,0 +1,4 @@ +# Индекс файлов S3 тоже должен пережить пересоздание контейнера. +[leveldb2] +enabled = true +dir = "/data/filerldb2" diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 0000000..8c9b83a --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,23 @@ +# Valkey для сессий lab2. Это единственное хранилище сессий, поэтому включена persistence. + +# Пароль (строка requirepass ...) лежит в файле секрета, который создаёт scripts/init_secrets.py. +include /run/secrets/valkey_auth + +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +# AOF: каждая запись попадает в журнал на диске, и после рестарта Valkey восстанавливает его. +# fsync на каждую запись: сессия подтверждается клиенту только после записи на диск. +# Записей мало (только вход и выход), поэтому цена fsync незаметна. +appendonly yes +appendfsync always + +# RDB-снимки не нужны: всё восстанавливается из AOF. +save "" + +# Сессии нельзя вытеснять молча: при нехватке памяти новая запись получит ошибку, +# и API ответит 503, а уже выданные сессии останутся. +maxmemory 128mb +maxmemory-policy noeviction diff --git a/migrations/0001_schema.sql b/migrations/0001_schema.sql new file mode 100644 index 0000000..bf00ee5 --- /dev/null +++ b/migrations/0001_schema.sql @@ -0,0 +1,46 @@ +-- Схема мессенджера: пользователи, чаты, участники, сообщения. +-- Ограничения повторяют правила контракта, чтобы база не приняла неверные данные, +-- даже если ошибка будет в коде приложения. + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, -- Argon2id, сам пароль не хранится + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Участники задаются при создании чата и потом не меняются. +-- position хранит порядок member_ids из запроса на создание. +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); + +-- Список чатов пользователя ищется по user_id. +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000), + -- Отправитель обязан состоять в чате: это проверяет и база, а не только код. + CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) +); diff --git a/migrations/0002_messages_history_index.sql b/migrations/0002_messages_history_index.sql new file mode 100644 index 0000000..d407eb1 --- /dev/null +++ b/migrations/0002_messages_history_index.sql @@ -0,0 +1,8 @@ +-- Индекс для чтения истории чата. Приложение выполняет запрос +-- SELECT ... FROM messages +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT $4 +-- Колонки индекса идут в том же порядке, поэтому страница читается диапазоном индекса +-- без сортировки и без просмотра всей истории чата. План на 10 000+ сообщениях — +-- в evidence/explain.txt (scripts/explain_history.py). +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0003_idempotency_keys.sql b/migrations/0003_idempotency_keys.sql new file mode 100644 index 0000000..52b4573 --- /dev/null +++ b/migrations/0003_idempotency_keys.sql @@ -0,0 +1,20 @@ +-- Идемпотентность отправки сообщения (с lab3). +-- Область ключа: пользователь + метод + путь чата + значение Idempotency-Key. +-- Строка ключа и сообщение создаются в одной транзакции: повтор запроса с тем же ключом +-- либо находит готовый результат, либо (если первая попытка не зафиксирована) создаёт его заново, +-- но никогда не создаёт второе сообщение. +-- Миграция только добавляет таблицу: версия API из lab2 продолжает работать с этой схемой. +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + method text NOT NULL, + path text NOT NULL, + key text NOT NULL, + request_hash text NOT NULL, -- sha256 разобранного тела: другое тело с тем же ключом — 409 + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, path, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$') +); + +-- Удаление ключей старше срока хранения (IDEMPOTENCY_TTL_SECONDS, не меньше 24 часов). +CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at); diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql new file mode 100644 index 0000000..d8e83f1 --- /dev/null +++ b/migrations/0004_attachments.sql @@ -0,0 +1,27 @@ +-- Байты только в S3; ключ не зависит от filename и не отдаётся клиенту. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + object_key text NOT NULL UNIQUE, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id), + UNIQUE (id, chat_id) +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id); +ALTER TABLE messages DROP CONSTRAINT messages_text_length; +ALTER TABLE messages ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000); +-- Массив хранит порядок attachment_ids и возвращается вместе с сообщением/повтором. +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + PRIMARY KEY (message_id, attachment_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); diff --git a/migrations/0005_async_attachments.sql b/migrations/0005_async_attachments.sql new file mode 100644 index 0000000..4e2dd48 --- /dev/null +++ b/migrations/0005_async_attachments.sql @@ -0,0 +1,91 @@ +-- PostgreSQL owns the logical job and its publication intent. The broker is a +-- durable delivery mechanism; it is never the only copy of accepted work. +ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1); +ALTER TABLE attachments ADD COLUMN error_code text + CHECK (error_code IS NULL OR error_code ~ '^[a-z][a-z0-9_]{0,63}$'); +ALTER TABLE attachments ADD COLUMN staging_key text; +UPDATE attachments SET staging_key = object_key; +ALTER TABLE attachments ALTER COLUMN staging_key SET NOT NULL; +ALTER TABLE attachments ADD COLUMN staging_cleanup_pending boolean NOT NULL DEFAULT false; +ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz; +ALTER TABLE attachments ADD CONSTRAINT attachments_result_state CHECK ( + (status <> 'ready' OR error_code IS NULL) AND + (status <> 'failed' OR error_code IS NOT NULL) AND + (NOT staging_cleanup_pending OR status = 'ready') +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments (id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + -- attempt is monotonic, including an explicit replay; its retry budget is separate. + attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0), + run_attempts integer NOT NULL DEFAULT 0 CHECK (run_attempts >= 0 AND run_attempts <= attempt), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + worker_id text, + lease_token uuid, + lease_until timestamptz, + error_code text CHECK (error_code IS NULL OR error_code ~ '^[a-z][a-z0-9_]{0,63}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((status = 'processing') = (worker_id IS NOT NULL AND lease_token IS NOT NULL AND lease_until IS NOT NULL)) +); +CREATE INDEX attachment_jobs_expired_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE status = 'failed'; + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attempt integer NOT NULL CHECK (attempt >= 1), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + worker_id text, + lease_token uuid, + lease_until timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, attempt), + CHECK ((lease_token IS NULL) = (lease_until IS NULL)), + CHECK ((lease_token IS NULL) = (worker_id IS NULL)) +); +CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox (available_at, created_at) + WHERE published_at IS NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); +CREATE INDEX attachment_staging_cleanup_idx ON attachments (updated_at) WHERE staging_cleanup_pending; + +CREATE TABLE job_audit ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + action text NOT NULL, + actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 120), + reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500), + details jsonb NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +-- Explicit operator fault hooks are persisted and consumed once across restarts. +CREATE TABLE job_faults ( + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + action text NOT NULL CHECK (action IN ('crash_after_objects', 'crash_after_ready', 'transient', 'permanent', 'stall_processing')), + remaining integer NOT NULL CHECK (remaining BETWEEN 0 AND 100), + PRIMARY KEY (job_id, action) +); + +-- An upgrade from lab4 also schedules its existing originals for the new pipeline. +INSERT INTO attachment_jobs (id, attachment_id) SELECT gen_random_uuid(), id FROM attachments; +INSERT INTO attachment_outbox (id, job_id, attempt) SELECT gen_random_uuid(), id, 1 FROM attachment_jobs; diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..8618a85 --- /dev/null +++ b/requirements.in @@ -0,0 +1,8 @@ +# Прямые зависимости API. Точные версии всех пакетов с хешами — в requirements.txt, +# он генерируется из этого файла командой из REPORT.md («Обновление зависимостей»). +psycopg[binary]==3.3.6 +psycopg-pool==3.3.3 +valkey==6.1.1 +argon2-cffi==25.1.0 +boto3==1.43.123 +Pillow==13.0.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5802a69 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,360 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in --generate-hashes --output-file requirements.txt --python C:\Users\Dima Chernikov\AppData\Local\Programs\Python\Python314\python.exe +argon2-cffi==25.1.0 \ + --hash=sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1 \ + --hash=sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741 + # via -r requirements.in +argon2-cffi-bindings==26.1.0 \ + --hash=sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2 \ + --hash=sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e \ + --hash=sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605 \ + --hash=sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a \ + --hash=sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8 \ + --hash=sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4 \ + --hash=sha256:1af817e84578ef8b7295ad17de0f9896e4c8520dbf2233c7aa5aa3d487256fc4 \ + --hash=sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba \ + --hash=sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb \ + --hash=sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2 \ + --hash=sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81 \ + --hash=sha256:242bb0cda2ae3650764fc194593d9ea45fc9e72729acd89778c7cfe184cec2a5 \ + --hash=sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29 \ + --hash=sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31 \ + --hash=sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8 \ + --hash=sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e \ + --hash=sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728 \ + --hash=sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a \ + --hash=sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35 \ + --hash=sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a \ + --hash=sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d \ + --hash=sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca \ + --hash=sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98 \ + --hash=sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1 \ + --hash=sha256:7014ab7e6f5d8511af92544667a0346ea6dfc314ea9a7cad1dba9fdb5c9a6e33 \ + --hash=sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36 \ + --hash=sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69 \ + --hash=sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1 \ + --hash=sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb \ + --hash=sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f \ + --hash=sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083 \ + --hash=sha256:b70225b5fd1e0d2ef4f7fd30d24658454535f0924dff0caca5dc08efbbbadfbb \ + --hash=sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08 \ + --hash=sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6 \ + --hash=sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440 \ + --hash=sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d \ + --hash=sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e \ + --hash=sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210 \ + --hash=sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990 \ + --hash=sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638 \ + --hash=sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4 + # via argon2-cffi +boto3==1.43.123 \ + --hash=sha256:055fabb990c5d2f31fce4ca569255bd640cdd11a3ca68c9ecad35f4f8a5b83b5 \ + --hash=sha256:d18d66e1e13faa0c25d0b1b8c5ce946e29fb5fc0b6456b655a93a678ff4cc313 + # via -r requirements.in +botocore==1.43.123 \ + --hash=sha256:80948cb9b1cdf1a1a25809089a42e3a1bc4af0258d198222db6e97dcc660dd0c \ + --hash=sha256:e30580f9c1282c321529238ff8ff75066096a2adbd0a293c8ae592c21d1733a3 + # via + # boto3 + # s3transfer +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via argon2-cffi-bindings +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore +pillow==13.0.0 \ + --hash=sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27 \ + --hash=sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782 \ + --hash=sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22 \ + --hash=sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15 \ + --hash=sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502 \ + --hash=sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692 \ + --hash=sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f \ + --hash=sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8 \ + --hash=sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13 \ + --hash=sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3 \ + --hash=sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369 \ + --hash=sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88 \ + --hash=sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d \ + --hash=sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb \ + --hash=sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f \ + --hash=sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0 \ + --hash=sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40 \ + --hash=sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599 \ + --hash=sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055 \ + --hash=sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033 \ + --hash=sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061 \ + --hash=sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca \ + --hash=sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f \ + --hash=sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca \ + --hash=sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c \ + --hash=sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4 \ + --hash=sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8 \ + --hash=sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620 \ + --hash=sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071 \ + --hash=sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc \ + --hash=sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a \ + --hash=sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6 \ + --hash=sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e \ + --hash=sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582 \ + --hash=sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f \ + --hash=sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79 \ + --hash=sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5 \ + --hash=sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683 \ + --hash=sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb \ + --hash=sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c \ + --hash=sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa \ + --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 \ + --hash=sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521 \ + --hash=sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e \ + --hash=sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c \ + --hash=sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058 \ + --hash=sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2 \ + --hash=sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463 \ + --hash=sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d \ + --hash=sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81 \ + --hash=sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf \ + --hash=sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63 \ + --hash=sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0 \ + --hash=sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42 \ + --hash=sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad \ + --hash=sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81 \ + --hash=sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c \ + --hash=sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4 \ + --hash=sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981 \ + --hash=sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43 \ + --hash=sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039 \ + --hash=sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76 \ + --hash=sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51 \ + --hash=sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2 \ + --hash=sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630 \ + --hash=sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac \ + --hash=sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510 \ + --hash=sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6 \ + --hash=sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3 \ + --hash=sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d \ + --hash=sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61 \ + --hash=sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932 \ + --hash=sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1 \ + --hash=sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d \ + --hash=sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00 \ + --hash=sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe \ + --hash=sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313 \ + --hash=sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806 \ + --hash=sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a \ + --hash=sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0 \ + --hash=sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52 \ + --hash=sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b \ + --hash=sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe + # via -r requirements.in +psycopg==3.3.6 \ + --hash=sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631 \ + --hash=sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2 + # via -r requirements.in +psycopg-binary==3.3.6 \ + --hash=sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781 \ + --hash=sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2 \ + --hash=sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475 \ + --hash=sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372 \ + --hash=sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de \ + --hash=sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03 \ + --hash=sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840 \ + --hash=sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79 \ + --hash=sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b \ + --hash=sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e \ + --hash=sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5 \ + --hash=sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9 \ + --hash=sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f \ + --hash=sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe \ + --hash=sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7 \ + --hash=sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138 \ + --hash=sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf \ + --hash=sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d \ + --hash=sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a \ + --hash=sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f \ + --hash=sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4 \ + --hash=sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6 \ + --hash=sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2 \ + --hash=sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300 \ + --hash=sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0 \ + --hash=sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a \ + --hash=sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6 \ + --hash=sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7 \ + --hash=sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc \ + --hash=sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e \ + --hash=sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30 \ + --hash=sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba \ + --hash=sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2 \ + --hash=sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22 \ + --hash=sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef \ + --hash=sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e \ + --hash=sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f \ + --hash=sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c \ + --hash=sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c \ + --hash=sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299 \ + --hash=sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e \ + --hash=sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638 \ + --hash=sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba \ + --hash=sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a \ + --hash=sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9 \ + --hash=sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc \ + --hash=sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2 \ + --hash=sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874 \ + --hash=sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c \ + --hash=sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e \ + --hash=sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312 \ + --hash=sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8 \ + --hash=sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac \ + --hash=sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18 \ + --hash=sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269 \ + --hash=sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb \ + --hash=sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10 \ + --hash=sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f \ + --hash=sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1 \ + --hash=sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784 \ + --hash=sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492 \ + --hash=sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc \ + --hash=sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52 \ + --hash=sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff \ + --hash=sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4 \ + --hash=sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8 + # via psycopg +psycopg-pool==3.3.3 \ + --hash=sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37 \ + --hash=sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d + # via -r requirements.in +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore +s3transfer==0.19.3 \ + --hash=sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553 \ + --hash=sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406 + # via boto3 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via psycopg-pool +tzdata==2026.6 \ + --hash=sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec \ + --hash=sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d + # via psycopg +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via botocore +valkey==6.1.1 \ + --hash=sha256:5880792990c6c2b5eb604a5ed5f98f300880b6dd92d123819b66ed54bb259731 \ + --hash=sha256:e2691541c6e1503b53c714ad9a35551ac9b7c0bbac93865f063dbc859a46de92 + # via -r requirements.in diff --git a/scripts/check_postgres_pause.py b/scripts/check_postgres_pause.py new file mode 100644 index 0000000..475ad4d --- /dev/null +++ b/scripts/check_postgres_pause.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Regression: pause PostgreSQL, direct API 503, discard connection and recover. + +No published replica ports: direct HTTP is sent from inside each API container. +PostgreSQL is always unpaused in finally. Uses existing local synthetic stand. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import queue +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +LOG = [] + + +def record(value): + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + LOG.append(text) + print(text, flush=True) + + +def compose(*args, timeout=40): + proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, + text=True, encoding='utf-8', errors='replace', timeout=timeout) + if proc.returncode: + raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:]) + return proc.stdout.strip() + + +PROBE = r''' +from concurrent.futures import ThreadPoolExecutor +import http.client, json, sys, time + +expected, count = map(int, sys.argv[1:]) +def request(path, status): + c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8) + start = time.monotonic() + try: + c.request('GET', path) + response = c.getresponse() + body = json.loads(response.read()) + assert response.status == status, (response.status, body) + assert response.getheader('X-Request-Id') + assert response.getheader('X-Instance-Id') + if status == 503: + assert body['error']['code'] == 'unavailable' + assert body['error']['request_id'] == response.getheader('X-Request-Id') + return round(time.monotonic() - start, 3) + finally: + c.close() + +with ThreadPoolExecutor(max_workers=count) as executor: + futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)] + live = request('/health/live', 200) + elapsed = [future.result() for future in futures] +print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live))) +''' + + +WORKER = r''' +import json, sys, time +from messenger.config import read_secret +from messenger.db import create_pool, Repository +from messenger.errors import Unavailable + +class SyntheticRollback(Exception): pass +pool = create_pool(read_secret('DATABASE_URL'), 1) +pool.open(wait=True, timeout=10) +repo = Repository(pool) +phase = sys.argv[1] +try: + try: + with repo._transaction() as conn: + conn.execute('SELECT 1') + print('ARMED', flush=True) + assert sys.stdin.readline().strip() == 'go' + started = time.monotonic() + if phase == 'query': + conn.execute('SELECT 1') + elif phase == 'rollback': + raise SyntheticRollback() + # commit: leave the block while PostgreSQL is paused. + except (Unavailable, SyntheticRollback) as error: + elapsed = time.monotonic() - started + assert conn.closed, 'timed-out connection was returned open' + assert elapsed < 8, elapsed + expected = SyntheticRollback if phase == 'rollback' else Unavailable + assert isinstance(error, expected), type(error) + print('TIMED_OUT', flush=True) + else: + raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused') + assert sys.stdin.readline().strip() == 'resume' + recovery = time.monotonic() + while True: + try: + repo.ping() + break + except Unavailable: + if time.monotonic() - recovery >= 15: + raise + time.sleep(0.1) + assert pool.get_stats().get('requests_waiting', 0) == 0 + print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3), + connection_closed=conn.closed, pool_max_size=1, + recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True) +finally: + pool.close(timeout=1) +''' + + +def line(proc, timeout=15): + result = queue.Queue() + threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start() + try: + return result.get(timeout=timeout).strip() + except queue.Empty: + raise AssertionError('database test worker did not finish in time') from None + + +def ready(c): + start = time.monotonic() + while time.monotonic() - start < 30: + try: + c.call('GET', '/health/ready', schema='Health') + return round(time.monotonic() - start, 3) + except Failure: + time.sleep(0.2) + raise AssertionError('API did not recover after PostgreSQL unpause') + + +def readiness(c, ids): + a, b, eve, chat = c.bootstrap() + for index in range(len(ids)): + compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1') + for cycle in range(2): + try: + compose('pause', 'postgres') + with ThreadPoolExecutor(max_workers=len(ids)) as executor: + futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api', + 'python', '-c', PROBE, '503', '12') for index in range(len(ids))] + for index, future in enumerate(futures): + record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result()))) + finally: + compose('unpause', 'postgres') + elapsed = ready(c) + # Same token and a real DB write/read after recovery, no API restart. + message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json'] + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json'] + check(message in page['items'], 'confirmed write not visible after unpause') + record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS')) + + +def operation(phase): + proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase], + cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding='utf-8', errors='replace') + try: + check(line(proc) == 'ARMED', 'worker could not open a transaction') + try: + compose('pause', 'postgres') + proc.stdin.write('go\n') + proc.stdin.flush() + check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase) + finally: + compose('unpause', 'postgres') + out, err = proc.communicate('resume\n', timeout=25) + check(proc.returncode == 0, 'worker failed: ' + err[-1200:]) + record(json.loads(out)) + finally: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + parser.add_argument('--output', type=Path) + args = parser.parse_args() + try: + c = Client(args.base_url, timeout=8) + ready(c) + ids = compose('ps', '-q', 'api').splitlines() + check(len(ids) >= 2, 'start at least two API replicas') + readiness(c, ids) + for phase in ('query', 'commit', 'rollback'): + operation(phase) + ready(c) + record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.') + return 0 + except Exception as error: + record('FAIL: ' + str(error)) + return 1 + finally: + if args.output: + args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 0000000..5c81113 --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages. + + python scripts/explain_history.py --output evidence/explain.txt + +The stand must be running. The script registers two synthetic users through the API, +creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and +prints the plans of the same query the API runs (src/messenger/db.py, list_messages): +first page, a page in the middle, the last page, the generic plan of the parameterized +query, and for comparison the middle page without the index (DROP INDEX inside a +transaction that is rolled back, so the database is left unchanged). +Standard library only. +""" +import argparse +import base64 +import json +from pathlib import Path +import subprocess +import sys +import time +import urllib.request +import uuid + +ROOT = Path(__file__).resolve().parents[1] +BASE = 'http://127.0.0.1:8080' +PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-X', '-v', 'ON_ERROR_STOP=1'] +PAGE = 51 # limit 50 + 1 row to learn whether the next page exists, as the API does + +lines = [] + + +def out(text=''): + lines.append(text) + print(text, flush=True) + + +def psql(sql, echo=True): + """Run SQL in one psql session; with echo the transcript shows every statement before its result.""" + args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-'] + done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + text = done.stdout.decode('utf-8', 'replace').strip() + if done.returncode != 0: + out(text) + raise SystemExit('psql failed') + return text + + +def api(method, path, body=None, headers=None): + headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode('utf-8') + headers['Content-Type'] = 'application/json' + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + with urllib.request.urlopen(request, timeout=10) as response: + return json.loads(response.read()) + + +def user(label): + username = label + '_' + uuid.uuid4().hex[:10] + password = 'Explain-' + uuid.uuid4().hex + created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password}) + credential = base64.b64encode((username + ':' + password).encode()).decode() + session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential}) + return created['id'], session['token'] + + +def page_sql(chat_id, after=None): + """The same SQL text as Repository.list_messages, with literal values.""" + sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id + if after: + sql += "AND (created_at, id) > ('%s', '%s') " % after + return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE + + +def main(): + parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages') + parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)') + parser.add_argument('--noise', type=int, default=20000, help='messages in another chat') + parser.add_argument('--output', default=None) + args = parser.parse_args() + sys.stdout.reconfigure(errors='replace') + + out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False))) + alice, alice_token = user('explain_a') + bob, _ = user('explain_b') + target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]}, + {'Authorization': 'Bearer ' + alice_token})['id'] + noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]}, + {'Authorization': 'Bearer ' + alice_token})['id'] + out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise)) + + started = time.monotonic() + psql(''' +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END, + 'seed message ' || i, clock_timestamp() + i * interval '1 millisecond' +FROM generate_series(1, %(messages)d) AS i; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond' +FROM generate_series(1, %(noise_count)d) AS i; +ANALYZE messages; +''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise}, + echo=False) + out('# inserted and analyzed in %.1f s' % (time.monotonic() - started)) + out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False)) + + def key_at(position): + row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1" + % (target, position - 1), echo=False) + return tuple(row.split('|')) + + middle, last = key_at(args.messages // 2), key_at(args.messages - 20) + explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) ' + out() + out('== 1. First page, middle page and last page: index range scan, no sort ==') + out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last)))) + out() + out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==') + out(psql('''SET plan_cache_mode = force_generic_plan; +PREPARE history_page(uuid, timestamptz, uuid, int) AS +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4; +%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE))) + out() + out('== 3. For comparison: the same middle page without the index (rolled back) ==') + out(psql('''BEGIN; +DROP INDEX messages_chat_history_idx; +%s%s; +ROLLBACK;''' % (explain, page_sql(target, middle)))) + out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False)) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..1ddbd99 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Create local secret files for the compose stand. + + python scripts/init_secrets.py + +Passwords are random and synthetic; they are created only if missing and never printed. +The URL/config files are always rebuilt from the password files, so the pair stays consistent. +secrets/ is listed in .gitignore and .dockerignore. + +PostgreSQL applies its password only when the data volume is created. If secrets/ is +deleted while the pgdata volume is kept, restore the old files or recreate the volume +(docker compose down -v deletes ALL lab data). +""" +from pathlib import Path +import json +import secrets + +ROOT = Path(__file__).resolve().parents[1] +DIR = ROOT / 'secrets' + + +def write(name, content): + path = DIR / name + path.write_text(content, encoding='utf-8', newline='\n') + # Readable for the non-root users inside containers (files are bind-mounted as is). + path.chmod(0o644) + + +def password(name): + path = DIR / name + if path.is_file() and path.read_text(encoding='utf-8').strip(): + print('keep secrets/' + name) + return path.read_text(encoding='utf-8').strip() + value = secrets.token_hex(24) # hex: safe inside URLs without escaping + write(name, value + '\n') + print('created secrets/' + name) + return value + + +def main(): + DIR.mkdir(exist_ok=True) + postgres = password('postgres_password') + valkey = password('valkey_password') + broker = password('broker_password') + write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres) + write('valkey_auth', 'requirepass %s\n' % valkey) + write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey) + write('broker_auth', 'requirepass %s\n' % broker) + write('broker_url', 'valkey://:%s@broker:6379/0\n' % broker) + admin_key = password('s3_admin_access_key_id') + admin_secret = password('s3_admin_secret_access_key') + api_key = password('s3_access_key_id') + api_secret = password('s3_secret_access_key') + worker_key = password('s3_worker_access_key_id') + worker_secret = password('s3_worker_secret_access_key') + identities = [ + dict(name='bootstrap', credentials=[dict(accessKey=admin_key, secretKey=admin_secret)], + actions=['Admin', 'Read', 'Write', 'List', 'Tagging']), + dict(name='messenger-api', credentials=[dict(accessKey=api_key, secretKey=api_secret)], + actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']), + dict(name='messenger-worker', credentials=[dict(accessKey=worker_key, secretKey=worker_secret)], + actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'])] + # Нет anonymous identity: ни объект, ни список bucket не доступны анонимно. + write('s3_config', json.dumps(dict(identities=identities)) + '\n') + print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url') + + +if __name__ == '__main__': + main() diff --git a/scripts/lab5_checks.py b/scripts/lab5_checks.py new file mode 100644 index 0000000..faa2e60 --- /dev/null +++ b/scripts/lab5_checks.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""Repeatable grade 3/4 failure checks for the local synthetic lab5 Compose stand. + +Run separately from other tests: scenarios deliberately restart/stop services. +Every scenario restores the original services in finally. No volumes, secrets or +application records are removed. Tokens live only in process memory. +""" +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + + +def command(*args): + result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', errors='replace') + if result.returncode: + # Never print environment, response bodies, credentials or arbitrary Docker logs. + raise Failure('command failed: ' + ' '.join(args[:4]) + ' (exit %d)' % result.returncode) + return result.stdout.strip() + + +def compose(*args): + return command('docker', 'compose', *args) + + +def sql(query): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query) + + +def rows(query): + value = sql("SELECT COALESCE(json_agg(t), '[]'::json) FROM (%s) t" % query) + return json.loads(value) + + +def identifier(value): + return str(uuid.UUID(value)) + + +def job(attachment_id): + data = rows("SELECT * FROM attachment_jobs WHERE attachment_id='%s'" % identifier(attachment_id)) + check(len(data) == 1, 'attachment must have exactly one logical job') + return data[0] + + +def audit(job_id): + return rows("SELECT action,actor,reason,details,created_at FROM job_audit WHERE job_id='%s' ORDER BY id" + % identifier(job_id)) + + +def cli(*args): + # The explicit operator service has broker/worker permissions. The API does + # not receive broker credentials merely to make an evidence script easier. + output = compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'jobs', *args) + return json.loads(output.splitlines()[-1]) if output else None + + +def inject(job_id, action, count=1): + return cli('inject', identifier(job_id), '--action', action, '--count', str(count), + '--actor', 'lab5-evidence', '--reason', 'repeatable synthetic grade 4 failure check') + + +def wait_until(predicate, description, seconds=180): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.4) + raise Failure('deadline exceeded: ' + description) + + +def ready(client): + def probe(): + try: + client.call('GET', '/health/ready', schema='Health') + return True + except Failure: + return False + wait_until(probe, 'API ready', 90) + + +def restore(client, *services): + # start preserves stopped container configuration and all named volumes. + compose('start', *services) + ready(client) + + +def upload(client, actor, chat, label): + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + label + '.png', + 202, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + attachment = result['json'] + check(attachment['status'] == 'queued', '202 must carry a queued snapshot') + check(attachment['variants'] == [] and attachment['error_code'] is None, 'queued result fields') + check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'accepted original checksum') + return attachment, content + + +def metadata(client, actor, attachment): + return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json'] + + +def wait_state(client, actor, attachment, status): + def probe(): + current = metadata(client, actor, attachment) + if current['status'] == status: + return current + check(current['status'] != 'failed' or status == 'failed', 'unexpected terminal failed state') + return wait_until(probe, 'attachment ' + status) + + +def verified_ready(client, actor, attachment, content): + result = wait_state(client, actor, attachment, 'ready') + check(result['id'] == attachment['id'], 'logical attachment identity changed') + check(result['error_code'] is None, 'ready error_code must be null') + check({item['name'] for item in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'grade 4 variants') + check(len(result['variants']) == 3, 'duplicate result variants') + for name in ['original', 'thumbnail', 'crop', 'optimized']: + data = client.call('GET', '/api/v1/attachments/' + result['id'] + '/content?variant=' + name, actor=actor) + record = result if name == 'original' else next(v for v in result['variants'] if v['name'] == name) + check(len(data['bytes']) == record['size_bytes'], 'published object size mismatch') + check(hashlib.sha256(data['bytes']).hexdigest() == record['sha256'], 'published object checksum mismatch') + if name == 'original': + check(data['bytes'] == content, 'original bytes changed') + count = int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % result['id'])) + check(count == 3, 'exactly three persistent variant records required') + return result + + +def object_state(attachment_id): + # Runs with the API's restricted bucket credentials; only selected facts return. + aid = identifier(attachment_id) + code = """import json +from messenger.config import read_secret +from messenger.db import create_pool +from messenger.objects import ObjectStore +from botocore.exceptions import ClientError +p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True) +with p.connection() as c: + r=c.execute('SELECT object_key,staging_key,staging_deleted_at,status,pipeline_version FROM attachments WHERE id=%%s',(%r,)).fetchone() +s=ObjectStore() +def exists(key): + try: s.client.head_object(Bucket=s.bucket,Key=key);return True + except ClientError as e: + if e.response['ResponseMetadata']['HTTPStatusCode']==404:return False + raise +prefix='variants/'+%r.replace('-','')+'/' +objects=s.client.list_objects_v2(Bucket=s.bucket,Prefix=prefix).get('Contents',[]) +print(json.dumps(dict(final=exists(r['object_key']),staging=exists(r['staging_key']),status=r['status'],staging_deleted=r['staging_deleted_at'] is not None,variant_objects=len(objects)))) +p.close() +""" % (aid, aid) + return json.loads(compose('exec', '-T', 'api', 'python', '-c', code).splitlines()[-1]) + + +def effect_snapshot(attachment_id): + aid = identifier(attachment_id) + return rows("SELECT a.status,a.object_key,a.sha256,a.updated_at,j.id AS job_id,j.attempt,j.updated_at AS job_updated_at," + "(SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id=a.id) AS variants " + "FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id='%s'" % aid)[0] + + +def fault_consumed(job_id, action): + return bool(rows("SELECT 1 AS consumed FROM job_faults WHERE job_id='%s' AND action='%s' AND remaining=0" + % (identifier(job_id), action))) + + +def stale_lease_fenced(attachment_id, old_job): + """A dead invocation cannot commit success or failure over the recovered result.""" + old_claim = dict(job_id=old_job['id'], lease_token=old_job['lease_token'], + attempt=old_job['attempt'], pipeline_version=old_job['pipeline_version']) + code = """import json +from messenger.config import read_secret +from messenger.db import create_pool +from messenger.jobs import Jobs +p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True);j=Jobs(p) +claim=json.loads(%r) +with p.connection() as c: + a=c.execute('SELECT object_key FROM attachments WHERE id=%%s',(%r,)).fetchone() + variants=c.execute('SELECT * FROM attachment_variants WHERE attachment_id=%%s ORDER BY name',(%r,)).fetchall() +assert j.finish(claim,a['object_key'],variants) is False,'old lease committed success' +assert j.fail(claim,'stale_worker',False,5,0) is False,'old lease committed failure' +p.close();print(json.dumps(dict(old_finish=False,old_fail=False))) +""" % (json.dumps(old_claim), identifier(attachment_id), identifier(attachment_id)) + return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code).splitlines()[-1]) + + +def stack(client): + config = json.loads(compose('config', '--format', 'json')) + services = config['services'] + for name in ('api', 'worker', 'dispatcher', 'postgres', 'valkey', 'broker', 's3'): + check(name in services, 'missing independent service ' + name) + check(not services[name].get('ports'), name + ': internal ports must not be published') + check(services['api']['deploy']['replicas'] >= 2, 'two API replicas required') + check(any(v.get('type') == 'volume' for v in services['broker'].get('volumes', [])), 'broker needs persistent volume') + check(services['worker'].get('deploy', {}).get('resources', {}).get('limits', {}).get('memory'), + 'worker memory limit required') + print('PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory', flush=True) + + +def durability(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'worker-down') + logical_job = job(attachment['id'])['id'] + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member) + message = client.message(member, chat, '', attachments=[attachment['id']])['json'] + check(message['attachment_ids'] == [attachment['id']], 'queued attachment cannot be linked') + check(object_state(attachment['id'])['staging'], '202 lost durable staging') + wait_until(lambda: bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NOT NULL" + % logical_job)), 'broker publication before restart') + compose('restart', 'api', 'broker') + ready(client) + check(metadata(client, member, attachment)['status'] == 'queued', 'worker-down status must remain queued') + history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=member, schema='MessagePage')['json'] + check(any(item['id'] == message['id'] for item in history['items']), 'linked message lost across restart') + compose('start', 'worker') + verified_ready(client, member, attachment, content) + check(job(attachment['id'])['id'] == logical_job, 'restart created a replacement logical job') + print('PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=' + logical_job, flush=True) + finally: + restore(client, 'api', 'broker', 'worker') + + +def outbox(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'dispatcher', 'worker') + attachment, content = upload(client, actor, chat, 'commit-publish-gap') + logical_job = job(attachment['id'])['id'] + pending = rows("SELECT attempt,published_at FROM attachment_outbox WHERE job_id='%s'" % logical_job) + check(len(pending) == 1 and pending[0]['published_at'] is None, 'COMMIT/publish gap was not isolated') + # API is killed after its successful COMMIT while publisher is absent. + compose('kill', '-s', 'SIGKILL', 'api') + compose('start', 'api') + ready(client) + check(job(attachment['id'])['id'] == logical_job, 'COMMIT intent lost after API crash') + compose('start', 'dispatcher', 'worker') + verified_ready(client, member, attachment, content) + check(job(attachment['id'])['id'] == logical_job, 'outbox recovery recreated the job') + print('PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=' + logical_job, flush=True) + finally: + restore(client, 'api', 'dispatcher', 'worker') + + +def broker_outage(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'broker', 'worker') + ready(client) + attachment, content = upload(client, actor, chat, 'broker-down') + logical_job = job(attachment['id'])['id'] + client.message(member, chat, 'broker outage does not stop messages', attachments=[attachment['id']]) + check(bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NULL" % logical_job)), + 'broker-down upload lacks durable outbox intent') + compose('start', 'broker', 'worker') + verified_ready(client, member, attachment, content) + print('PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=' + logical_job, flush=True) + finally: + restore(client, 'broker', 'dispatcher', 'worker') + + +def wait_delivery_ack(message_id): + code = """import json +from messenger.broker import Broker +b=Broker();target=tuple(map(int,%r.split('-'))) +groups=b.client.xinfo_groups(b.stream) +group=next((g for g in groups if g['name']==b.group),None) +delivered=bool(group and tuple(map(int,group['last-delivered-id'].split('-')))>=target) +pending=b.client.xpending_range(b.stream,b.group,%r,%r,1) +print(json.dumps(dict(done=delivered and not pending)));b.close() +""" % (message_id, message_id, message_id) + wait_until(lambda: json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code))['done'], 'redelivery ack') + + +def redelivery(client): + actor, member, _, chat = client.bootstrap() + attachment, content = upload(client, actor, chat, 'redelivery') + completed = verified_ready(client, member, attachment, content) + before = effect_snapshot(attachment['id']) + published = cli('redeliver', before['job_id']) + message_id = published['message_id'] + wait_delivery_ack(message_id) + after = effect_snapshot(attachment['id']) + check(before == after, 'ready duplicate mutated persistent result, job attempt or timestamps') + check(metadata(client, member, attachment) == completed, 'ready duplicate changed public metadata') + print('PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=' + before['job_id'], flush=True) + + +def crash(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'crash-after-objects') + logical_job = job(attachment['id'])['id'] + inject(logical_job, 'crash_after_objects') + compose('start', 'worker') + wait_until(lambda: fault_consumed(logical_job, 'crash_after_objects'), 'crash fault consumed') + at_crash = object_state(attachment['id']) + check(at_crash['final'] and at_crash['variant_objects'] == 3, + 'injected crash occurred before all four final objects existed') + check(at_crash['status'] == 'processing' and at_crash['staging'], + 'crash window must precede ready commit and retain staging') + check(not rows("SELECT 1 FROM attachment_variants WHERE attachment_id='%s'" % attachment['id']), + 'crash window must precede publication of variant metadata') + # The durable hook is consumed before exit, so container restart cannot loop it. + compose('start', 'worker') + verified_ready(client, member, attachment, content) + result = job(attachment['id']) + check(result['id'] == logical_job and result['attempt'] >= 2, 'crashed job did not recover its expired lease') + check(object_state(attachment['id'])['variant_objects'] == 3, 'redelivery duplicated S3 variants') + print('PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=%s attempts=%s window=%s' + % (logical_job, result['attempt'], json.dumps(at_crash, sort_keys=True)), flush=True) + finally: + restore(client, 'worker') + + +def lease(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'expired-processing') + logical_job = job(attachment['id'])['id'] + inject(logical_job, 'stall_processing') + compose('start', 'worker') + wait_until(lambda: fault_consumed(logical_job, 'stall_processing'), 'processing stall hook') + held = job(attachment['id']) + check(held['status'] == 'processing' and held['lease_until'], 'stall did not hold a processing lease') + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member) + compose('kill', '-s', 'SIGKILL', 'worker') + compose('start', 'worker') + verified_ready(client, member, attachment, content) + recovered = job(attachment['id']) + check(recovered['id'] == logical_job and recovered['attempt'] > held['attempt'], 'expired processing was not reclaimed') + snapshot = effect_snapshot(attachment['id']) + stale_lease_fenced(attachment['id'], held) + check(effect_snapshot(attachment['id']) == snapshot, 'stale lease changed the recovered ready result') + print('PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=%s old_lease_until=%s attempts=%d' + % (logical_job, held['lease_until'], recovered['attempt']), flush=True) + print('PASS fencing: old lease cannot finish or fail after recovered ready commit', flush=True) + finally: + restore(client, 'worker') + + +def failed_contract(client, actor, chat, attachment): + failed = wait_state(client, actor, attachment, 'failed') + check(bool(re.fullmatch(r'[a-z][a-z0-9_]{0,63}', failed['error_code'] or '')), 'safe terminal error code required') + client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor, + body={'text': '', 'attachment_ids': [attachment['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex}) + return failed + + +def retries(client): + actor, member, _, chat = client.bootstrap() + config = json.loads(compose('config', '--format', 'json'))['services']['worker'].get('environment', {}) + max_attempts = int(config.get('JOB_MAX_ATTEMPTS', 5)) + retry_base = float(config.get('JOB_RETRY_BASE_SECONDS', 1)) + retry_max = float(config.get('JOB_RETRY_MAX_SECONDS', 30)) + check(max_attempts >= 3, 'retry demonstration requires a budget of at least three attempts') + try: + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'transient-retry') + logical_job = job(attachment['id'])['id'] + inject(logical_job, 'transient', 2) + compose('start', 'worker') + verified_ready(client, member, attachment, content) + retried = job(attachment['id']) + check(retried['attempt'] >= 2, 'transient failure was not retried') + events = audit(logical_job) + retry_events = [event for event in events if event['action'] == 'retry' and event['reason'] == 'injected_transient'] + check(len(retry_events) == 2, 'both retry delays must be observable in durable audit') + randomized = False + for event in retry_events: + details = event['details'] + cap = min(retry_max, retry_base * 2.0 ** (details['run_attempts'] - 1)) + check(cap / 2 <= details['delay_seconds'] <= cap, 'retry is outside its equal-jitter exponential interval') + randomized |= details['delay_seconds'] not in (cap / 2, cap) + created = dt.datetime.fromisoformat(event['created_at'].replace('Z', '+00:00')) + available = dt.datetime.fromisoformat(details['available_at'].replace('Z', '+00:00')) + check(abs((available - created).total_seconds() - details['delay_seconds']) < 0.25, + 'retry due time does not persist the chosen jitter delay') + check(randomized, 'retry evidence contains no randomized delay') + print('PASS transient errors retry with durable backoff/jitter; job_id=%s audit=%s' + % (logical_job, json.dumps(retry_events, sort_keys=True)), flush=True) + + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'retry-exhausted') + logical_job = job(attachment['id'])['id'] + inject(logical_job, 'transient', max_attempts) + compose('start', 'worker') + failed = failed_contract(client, member, chat, attachment) + check(failed['error_code'] == 'retry_exhausted', 'exhausted job needs a distinct terminal error code') + exhausted = job(attachment['id']) + check(exhausted['run_attempts'] == max_attempts, 'retry budget not enforced') + check(fault_consumed(logical_job, 'transient'), 'retry fault count does not match exhausted attempts') + failed_jobs = cli('failed', '--limit', '1000') + check(any(item['job_id'] == logical_job for item in failed_jobs), 'failed job missing from operator registry') + check(object_state(attachment['id'])['staging'], 'failed job lost original needed for replay') + cli('cleanup') + check(object_state(attachment['id'])['staging'], 'cleanup removed failed input required for replay') + replay = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'operator resolved synthetic failure') + check(replay['job_id'] == logical_job, 'operator replay created a different logical job') + verified_ready(client, member, attachment, content) + before = effect_snapshot(attachment['id']) + noop = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'verify successful replay is a no-op') + check(effect_snapshot(attachment['id']) == before, 'replay re-executed a successful job') + actions = {event['action'] for event in audit(logical_job) if event['actor'] == 'lab5-evidence'} + check({'replay', 'replay_noop'} <= actions, 'explicit replay and successful no-op need operator audit records') + print('PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=%s max_attempts=%d error_code=%s replay=%s' + % (logical_job, max_attempts, failed['error_code'], json.dumps(noop, sort_keys=True)), flush=True) + finally: + restore(client, 'worker') + + +def permanent(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'worker') + attachment, _ = upload(client, actor, chat, 'permanent-failure') + logical_job = job(attachment['id'])['id'] + inject(logical_job, 'permanent') + compose('start', 'worker') + failed = failed_contract(client, member, chat, attachment) + check(failed['error_code'] == 'invalid_image', 'permanent errors need a distinct safe error code') + check(job(attachment['id'])['run_attempts'] == 1, 'permanent errors must not consume all retries') + print('PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=%s error_code=%s' + % (logical_job, failed['error_code']), flush=True) + finally: + restore(client, 'worker') + + +def staging(client): + actor, member, _, chat = client.bootstrap() + try: + compose('stop', 'worker') + attachment, content = upload(client, actor, chat, 'staging-safety') + before = object_state(attachment['id']) + check(before['staging'] and not before['final'], 'staging/final publication boundary is not visible') + cli('cleanup') + check(object_state(attachment['id']) == before, 'cleanup removed current queued staging') + compose('start', 'worker') + verified_ready(client, member, attachment, content) + cli('cleanup') + after = wait_until(lambda: (lambda s: s if not s['staging'] and s['staging_deleted'] else None) + (object_state(attachment['id'])), 'post-commit staging cleanup') + check(after['final'] and after['status'] == 'ready', 'cleanup removed committed original') + print('PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=' + attachment['id'], flush=True) + finally: + restore(client, 'worker') + + +SCENARIOS = ['stack', 'durability', 'outbox', 'broker_outage', 'redelivery', 'crash', 'lease', 'retries', 'permanent', 'staging'] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080')) + parser.add_argument('--only', nargs='+', choices=SCENARIOS) + args = parser.parse_args() + client = Client(args.base_url, timeout=15) + ready(client) + for name in args.only or SCENARIOS: + ready(client) + print('START ' + name + ' ' + dt.datetime.now(dt.timezone.utc).isoformat(), flush=True) + globals()[name](client) + print('PASS selected lab5 grade 3/4 checks; no grade 5 track claimed', flush=True) + + +if __name__ == '__main__': + try: + main() + except (Failure, ValueError, OSError) as exc: + print('FAIL: ' + str(exc), flush=True) + sys.exit(1) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..65dda8e --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,10 @@ +@echo off +rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly, +rem and Windows cannot execute a .sh file. Same actions, volumes are kept. +setlocal +cd /d "%~dp0.." +docker compose stop api worker dispatcher || exit /b 1 +docker compose restart postgres valkey s3 broker || exit /b 1 +docker compose start api dispatcher worker || exit /b 1 +docker compose restart lb || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..950d386 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Hook for: make test-persistence ACTION_SCRIPT=scripts/restart.sh +# Restarts the whole stand: all API replicas, PostgreSQL, Valkey and the load balancer, +# keeping the named volumes: no "down -v", no FLUSHALL, no re-seed. +# The test itself waits for API readiness through the load balancer afterwards. +set -eu +cd "$(dirname "$0")/.." +docker compose stop api worker dispatcher +docker compose restart postgres valkey s3 broker +docker compose start api dispatcher worker +docker compose restart lb diff --git a/scripts/stack_evidence.py b/scripts/stack_evidence.py new file mode 100644 index 0000000..227dcc1 --- /dev/null +++ b/scripts/stack_evidence.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Sanitized runtime snapshot: real processes, start times, limits and volumes; no environment/secrets.""" +import datetime as dt +import json +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8') + + +def main(): + ids = command('docker', 'compose', 'ps', '-a', '-q').split() + containers = json.loads(command('docker', 'inspect', *ids)) + result = {'captured_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'containers': []} + for item in containers: + # Only these selected fields are persisted; Config.Env is deliberately excluded. + result['containers'].append(dict( + name=item['Name'].lstrip('/'), image=item['Config']['Image'], + status=item['State']['Status'], started_at=item['State']['StartedAt'], + health=item['State'].get('Health', {}).get('Status'), + memory_limit_bytes=item['HostConfig']['Memory'], cpu_limit_nanos=item['HostConfig']['NanoCpus'], + volumes=[dict(name=m.get('Name'), destination=m['Destination']) + for m in item['Mounts'] if m['Type'] == 'volume'], + published_ports=item['NetworkSettings']['Ports'])) + text = json.dumps(result, indent=2) + (ROOT / 'evidence/stack.json').write_text(text + '\n', encoding='utf-8') + print('Saved evidence/stack.json (process start times, versions, limits, volumes and ports)') + + +if __name__ == '__main__': + main() diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..d91d84b --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,19 @@ +@echo off +rem Windows twin of scripts/stop-one.sh: tests/resilience.py starts the hook directly, +rem and Windows cannot execute a .sh file. Stops ONE api replica by X-Instance-Id (container ID). +setlocal +if "%TARGET_INSTANCE_ID%"=="" ( + echo TARGET_INSTANCE_ID is not set 1>&2 + exit /b 2 +) +cd /d "%~dp0.." +set PROJECT=%COMPOSE_PROJECT_NAME% +if "%PROJECT%"=="" set PROJECT=mtusi-lab5 +set ID= +for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=%PROJECT%" --filter "label=com.docker.compose.service=api"') do set ID=%%i +if "%ID%"=="" ( + echo TARGET_INSTANCE_ID is not a running api replica of %PROJECT% 1>&2 + exit /b 1 +) +docker stop %ID% >nul || exit /b 1 +exit /b 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..f657d68 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Hook for: make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# Stops exactly ONE API replica and leaves it stopped. TARGET_INSTANCE_ID is the X-Instance-Id +# of the replica, which is its container hostname = short container ID. The hook checks that +# the ID belongs to a running "api" container of this compose project before stopping it. +# docker stop sends SIGTERM: the replica reports not ready, HAProxy removes it, then it exits. +# Restore afterwards: docker compose up -d (or docker start ) +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +project="${COMPOSE_PROJECT_NAME:-mtusi-lab5}" +id=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \ + --filter "label=com.docker.compose.project=$project" \ + --filter "label=com.docker.compose.service=api") +if [ -z "$id" ]; then + echo "TARGET_INSTANCE_ID is not a running api replica of $project" >&2 + exit 1 +fi +docker stop "$id" >/dev/null diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..36c2555 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Run reproducible acceptance checks against an already running local lab5 stand. + +The default checks do not stop services. --faults explicitly includes the grade 4 +fault suite; each fault restores services without deleting volumes or secrets. +""" +import argparse +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + + +def wait_ready(url): + client = Client(url) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call('GET', '/health/ready', schema='Health') + return + except Failure: + time.sleep(0.5) + raise Failure('API did not become ready before verification block') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080')) + parser.add_argument('--faults', action='store_true', help='Also run disruptive grade 4 fault checks') + parser.add_argument('--only', nargs='+', choices=['offline', 'unit', 'smoke', 'tests', 'lab5-checks', + 'persistence', 'failover', 'postgres-pause', 'cleanup', 'stack']) + parser.add_argument('--output-prefix', default='', help='Keep a separate set of UTF-8 evidence logs') + options = parser.parse_args() + if Path(options.output_prefix).name != options.output_prefix or any(c in options.output_prefix for c in '/\\'): + parser.error('output prefix must be a filename prefix') + checks = [ + ('offline', ['scripts/check_contract.py']), + ('unit', ['docker', 'compose', 'run', '--rm', '--no-deps', '--entrypoint', 'python', + '-v', './unit-tests:/app/unit-tests:ro', 'worker', '-m', 'unittest', 'discover', '-s', 'unit-tests', '-v']), + ('smoke', ['tests/smoke.py', '--base-url', options.base_url]), + ('tests', ['-m', 'unittest', 'discover', '-s', 'student-tests', '-v']), + ('cleanup', ['docker', 'compose', '--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup']), + ('stack', ['scripts/stack_evidence.py']), + ] + if options.faults or (options.only and 'lab5-checks' in options.only): + checks.insert(4, ('lab5-checks', ['scripts/lab5_checks.py', '--base-url', options.base_url])) + for name, mode, hook in ( + ('persistence', 'persistence', 'restart'), ('failover', 'failover', 'stop-one') + ): + if options.faults or (options.only and name in options.only): + script = 'scripts/%s.%s' % (hook, 'cmd' if os.name == 'nt' else 'sh') + checks.insert(-1, (name, ['tests/resilience.py', '--mode', mode, '--action-script', script, + '--base-url', options.base_url])) + if options.faults or (options.only and 'postgres-pause' in options.only): + checks.insert(-1, ('postgres-pause', ['scripts/check_postgres_pause.py', '--base-url', options.base_url])) + (ROOT / 'evidence').mkdir(exist_ok=True) + for name, args in checks: + if options.only and name not in options.only: + continue + if name != 'offline': + wait_ready(options.base_url) + invocation = args if name in ('unit', 'cleanup') else [sys.executable, '-X', 'utf8', *args] + result = subprocess.run(invocation, cwd=ROOT, + env=dict(os.environ, BASE_URL=options.base_url), capture_output=True) + if name == 'failover': + restored = subprocess.run(['docker', 'compose', 'up', '-d', '--wait'], cwd=ROOT, + capture_output=True) + if restored.returncode: + print('Failed to restore API replica after failover', flush=True) + return restored.returncode + content = result.stdout.decode('utf-8', errors='replace') + result.stderr.decode('utf-8', errors='replace') + (ROOT / 'evidence' / (options.output_prefix + name + '.txt')).write_text(content, encoding='utf-8') + print(name + ': ' + ('PASS' if result.returncode == 0 else 'FAIL'), flush=True) + if result.returncode: + print(content[-6000:], flush=True) + return result.returncode + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 0000000..69f8a78 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1 @@ +"""In-memory мессенджер, лабораторная 1. Только стандартная библиотека Python.""" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 0000000..c004a6d --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,94 @@ +"""Точка входа API: python -m messenger.""" +import logging +import signal +import sys +import threading +import time + +from .config import load_config +from .db import Repository, create_pool +from .server import MessengerServer +from .sessions import SessionStore +from .objects import ObjectStore + +log = logging.getLogger('messenger') + +# Как часто реплика удаляет ключи идемпотентности старше срока хранения. +CLEANUP_INTERVAL = 600 + + +def cleanup_idempotency_keys(repo, ttl, stop): + """Фоновая очистка. Каждая реплика делает одно и то же; повторный DELETE безвреден.""" + while not stop.wait(CLEANUP_INTERVAL): + try: + deleted = repo.delete_expired_idempotency_keys(ttl) + log.info('event=idempotency_cleanup deleted=%d', deleted) + except Exception as error: # очистка не должна останавливать реплику + log.warning('event=idempotency_cleanup_failed error=%s', type(error).__name__) + + +def main(): + config = load_config() + logging.Formatter.converter = time.gmtime # время в логах — UTC + logging.basicConfig( + stream=sys.stdout, + level=config.log_level, + format='%(asctime)s.%(msecs)03dZ %(levelname)s %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S', + ) + # Пул открывается в фоне: если БД ещё не готова, API всё равно стартует, + # а /health/ready отвечает 503, пока соединение не появится. + pool = create_pool(config.database_url, config.db_pool_size) + pool.open(wait=False) + sessions = SessionStore(config.cache_url, config.session_ttl) + repo = Repository(pool) + try: + server = MessengerServer((config.host, config.port), repo, sessions, config.instance_id, + config.idempotency_ttl, ObjectStore()) + except OSError as error: + log.error('event=bind_failed host=%s port=%d error=%s', config.host, config.port, error) + return 1 + + # Обработчик сигнала только запоминает номер: всю остановку делает главный поток. + received = [] + + def on_signal(signum, _frame): + received.append(signum) + + for name in ('SIGTERM', 'SIGINT', 'SIGBREAK'): # SIGBREAK есть только в Windows + if hasattr(signal, name): + signal.signal(getattr(signal, name), on_signal) + + stop_cleanup = threading.Event() + threading.Thread(target=cleanup_idempotency_keys, args=(repo, config.idempotency_ttl, stop_cleanup), + name='idempotency-cleanup', daemon=True).start() + threading.Thread(target=server.serve_forever, name='accept-loop', daemon=True).start() + log.info('event=started instance=%s version=%s host=%s port=%d env=%s session_ttl_s=%d db_pool_max=%d ' + 'drain_delay_s=%g shutdown_timeout_s=%g', config.instance_id, config.version, config.host, config.port, + config.env, config.session_ttl, config.db_pool_size, config.drain_delay, config.shutdown_timeout) + + while not received: + time.sleep(0.1) + + started = time.monotonic() + log.info('event=shutdown_started signal=%s', signal.Signals(received[0]).name) + # 1. Реплика объявляет себя неготовой, но ещё обслуживает запросы: балансировщик проверяет + # /health/ready каждую секунду и после двух неудач (около 2 с) перестаёт слать сюда трафик. + server.begin_not_ready() + time.sleep(config.drain_delay) + # 2. Приём закрывается. Запросы на уже открытых соединениях получают 503. + server.begin_drain() + server.shutdown() + server.server_close() + # 3. Начатые запросы дорабатывают. Секунда остаётся на закрытие соединений с хранилищами. + remaining = config.shutdown_timeout - 1 - (time.monotonic() - started) + clean = server.wait_idle(max(remaining, 0)) + stop_cleanup.set() + pool.close(timeout=1) + sessions.close() + log.info('event=shutdown_finished clean=%s duration_ms=%.0f', clean, (time.monotonic() - started) * 1000) + return 0 if clean else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/broker.py b/src/messenger/broker.py new file mode 100644 index 0000000..72f4711 --- /dev/null +++ b/src/messenger/broker.py @@ -0,0 +1,95 @@ +"""Persistent Valkey Streams transport; schema fields contain IDs and metadata only.""" +import datetime as dt +import json +import os +import re +import uuid + +import valkey +from valkey.exceptions import ResponseError + +from .config import read_secret + +REQUIRED = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} +OPTIONAL = {'traceparent', 'tracestate'} + + +def validate_message(payload): + """Keep the wire contract strict even without a JSON Schema runtime dependency.""" + if not isinstance(payload, dict) or not REQUIRED <= payload.keys() or payload.keys() - REQUIRED - OPTIONAL: + raise ValueError('invalid job fields') + if type(payload['schema_version']) is not int or payload['schema_version'] != 1: + raise ValueError('invalid schema version') + if payload['type'] != 'attachment.process.v1': + raise ValueError('invalid job type') + for name in ('job_id', 'attachment_id'): + if not isinstance(payload[name], str): + raise ValueError('invalid identifier') + uuid.UUID(payload[name]) + for name in ('attempt', 'pipeline_version'): + if type(payload[name]) is not int or payload[name] < 1: + raise ValueError('invalid positive integer') + created = payload['created_at'] + if not isinstance(created, str) or dt.datetime.fromisoformat(created.replace('Z', '+00:00')).tzinfo is None: + raise ValueError('invalid created_at') + if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or + re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None): + raise ValueError('invalid traceparent') + if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512): + raise ValueError('invalid tracestate') + return payload + + +class InvalidDelivery(Exception): + def __init__(self, message_id): + self.message_id = message_id + super().__init__('invalid broker job') + + +class Broker: + def __init__(self): + self.stream = os.environ.get('JOB_STREAM', 'attachment-jobs') + self.group = os.environ.get('JOB_CONSUMER_GROUP', 'attachment-workers') + self.client = valkey.Valkey.from_url(read_secret('BROKER_URL'), decode_responses=True, + socket_connect_timeout=2, socket_timeout=4, + retry_on_timeout=False, health_check_interval=15) + + def ensure_group(self): + try: + self.client.xgroup_create(self.stream, self.group, id='0', mkstream=True) + except ResponseError as error: + if not str(error).startswith('BUSYGROUP'): + raise + + def publish(self, payload): + validate_message(payload) + # Broker runs AOF with appendfsync=always: successful XADD is its durable + # acceptance boundary. A lost response can cause a harmless duplicate. + return self.client.xadd(self.stream, {'payload': json.dumps(payload, separators=(',', ':'))}) + + def read(self, consumer, idle_ms): + # COUNT 1 bounds prefetch. Dead consumer pending entries are claimed only + # after the DB lease duration; DB fencing still decides who may commit. + pending = self.client.xautoclaim(self.stream, self.group, consumer, idle_ms, + start_id='0-0', count=1) + messages = pending[1] + if not messages: + entries = self.client.xreadgroup(self.group, consumer, {self.stream: '>'}, count=1, block=1000) + messages = entries[0][1] if entries else [] + if not messages: + return None + message_id, fields = messages[0] + try: + if set(fields) != {'payload'} or len(fields['payload']) > 4096: + raise ValueError('invalid envelope') + return message_id, validate_message(json.loads(fields['payload'])) + except (ValueError, TypeError, KeyError, AttributeError): + # Never acknowledge malformed work without a durable effect. Only + # our validated dispatcher can write to the private broker stream. + raise InvalidDelivery(message_id) from None + + def ack(self, message_id): + self.client.xack(self.stream, self.group, message_id) + + def close(self): + self.client.close() diff --git a/src/messenger/bucket.py b/src/messenger/bucket.py new file mode 100644 index 0000000..190e95b --- /dev/null +++ b/src/messenger/bucket.py @@ -0,0 +1,31 @@ +"""Идемпотентное создание bucket отдельными административными credentials.""" +import time + +from botocore.exceptions import BotoCoreError, ClientError + +from .objects import ObjectStore + + +def main(): + store = ObjectStore(admin=True) + for attempt in range(30): + try: + store.client.create_bucket(Bucket=store.bucket, ACL='private') + print('bucket ready (private; authentication configured by s3_config)') + return + except ClientError as error: + if error.response['Error']['Code'] in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'): + # Нельзя присваивать bucket другого владельца: head с собственным ключом. + store.ping() + print('bucket already ready') + return + if attempt == 29: + raise + except BotoCoreError: + if attempt == 29: + raise + time.sleep(1) + + +if __name__ == '__main__': + main() diff --git a/src/messenger/cleanup_objects.py b/src/messenger/cleanup_objects.py new file mode 100644 index 0000000..4c353ee --- /dev/null +++ b/src/messenger/cleanup_objects.py @@ -0,0 +1,84 @@ +"""Сборщик неучтённых объектов; default dry-run, grace >= 1 час, повторный запуск безопасен. + +Оператор: docker compose --profile tools run --rm cleanup [--apply] +Проверяются originals/, staging/ и variants/. Свежий объект и текущая работа защищены. Проверка БД и удаление +делаются под той же advisory lock, которую upload держит от PUT до COMMIT. +При недоступности БД/S3 обход прерывается, никаких delete по неполному снимку. +""" +import argparse +import datetime as dt +import json +import os +import re +import time + +from .config import read_secret +from .db import create_pool +from .objects import ObjectStore + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + parser.add_argument('--failed-retention-seconds', type=int, + default=int(os.environ.get('STAGING_RETENTION_SECONDS', '86400'))) + args = parser.parse_args() + if args.grace_seconds < 3600: + parser.error('grace не меньше 3600 секунд') + if args.failed_retention_seconds < 86400: + parser.error('retention failed не меньше 86400 секунд') + store = ObjectStore(admin=True) + pool = create_pool(read_secret('DATABASE_URL'), 1) + pool.open(wait=True) + cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds) + failed_cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.failed_retention_seconds) + counts = dict(scanned=0, fresh=0, referenced=0, orphan=0, deleted=0) + start = time.monotonic() + try: + paginator = store.client.get_paginator('list_objects_v2') + pages = (page for prefix in ('originals/', 'staging/', 'variants/') + for page in paginator.paginate(Bucket=store.bucket, Prefix=prefix)) + for page in pages: + for obj in page.get('Contents', []): + counts['scanned'] += 1 + if obj['LastModified'] >= cutoff: + counts['fresh'] += 1 + continue + retired = False + with pool.connection() as conn: + conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (obj['Key'],)) + row = conn.execute( + 'SELECT id,status,updated_at,staging_key FROM attachments ' + 'WHERE object_key=%s OR staging_key=%s FOR UPDATE', + (obj['Key'], obj['Key'])).fetchone() + if row and row['staging_key'] == obj['Key'] and row['status'] == 'failed' and row['updated_at'] < failed_cutoff: + counts['orphan'] += 1 + if args.apply: + # Сначала durable tombstone под той же row lock, что replay. + # После COMMIT replay уже запрещён даже при crash до S3 DELETE. + conn.execute('UPDATE attachments SET staging_deleted_at=clock_timestamp(), ' + 'staging_cleanup_pending=false WHERE id=%s', (row['id'],)) + retired = True + else: + # Частичный результат активной job ещё не имеет variant metadata. + match = re.fullmatch(r'variants/([0-9a-f]{32})/v[0-9]+/[^/]+', obj['Key']) + if not row and match: + row = conn.execute('SELECT 1 FROM attachments WHERE id=%s::uuid', (match[1],)).fetchone() + if row: + counts['referenced'] += 1 + else: + counts['orphan'] += 1 + if args.apply: + store.call('delete_object', Key=obj['Key']) + counts['deleted'] += 1 + if retired: + store.call('delete_object', Key=obj['Key']) + counts['deleted'] += 1 + finally: + pool.close() + print(json.dumps(dict(counts, apply=args.apply, seconds=round(time.monotonic()-start, 3)))) + + +if __name__ == '__main__': + main() diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 0000000..c0d787b --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,84 @@ +"""Конфигурация из переменных окружения и файлов секретов (см. .env.example и compose.yaml).""" +import os +import socket +from collections import namedtuple + +Config = namedtuple('Config', 'host port env log_level shutdown_timeout drain_delay session_ttl idempotency_ttl ' + 'db_pool_size database_url cache_url instance_id version') + +LOG_LEVELS = ('DEBUG', 'INFO', 'WARNING', 'ERROR') +# Заявленный срок остановки по SIGTERM — не более 10 секунд. +MAX_SHUTDOWN_TIMEOUT = 10.0 +MAX_SESSION_TTL = 30 * 24 * 3600 +# Контракт: результат идемпотентного запроса хранится не меньше 24 часов. +MIN_IDEMPOTENCY_TTL = 24 * 3600 + + +def _environ(environ): + # strip(): .env из Windows может принести в значение символ \r. + return {name: value.strip() for name, value in (os.environ if environ is None else environ).items()} + + +def read_secret(name, environ=None): + """Секрет из файла, путь к которому в NAME_FILE (секреты Docker), или из NAME для отладки. + + Значение секрета никогда не попадает в сообщения об ошибках и в логи. + """ + environ = _environ(environ) + path = environ.get(name + '_FILE') + if path: + try: + with open(path, encoding='utf-8') as file: + value = file.read().strip() + except OSError as error: + raise SystemExit('Не удалось прочитать файл из %s_FILE: %s' % (name, error.strerror)) + else: + value = environ.get(name, '') + if not value: + raise SystemExit('Нужен %s_FILE (путь к файлу секрета) или %s' % (name, name)) + return value + + +def _integer(environ, name, default, low, high): + try: + value = int(environ.get(name, str(default))) + except ValueError: + raise SystemExit('%s должен быть целым числом' % name) + if not low <= value <= high: + raise SystemExit('%s должен быть в диапазоне %d..%d' % (name, low, high)) + return value + + +def load_config(environ=None): + environ = _environ(environ) + try: + shutdown_timeout = float(environ.get('APP_SHUTDOWN_TIMEOUT', '8')) + drain_delay = float(environ.get('APP_DRAIN_DELAY', '3')) + except ValueError: + raise SystemExit('APP_SHUTDOWN_TIMEOUT и APP_DRAIN_DELAY должны быть числами') + if not 0 < shutdown_timeout <= MAX_SHUTDOWN_TIMEOUT: + raise SystemExit('APP_SHUTDOWN_TIMEOUT должен быть больше 0 и не больше 10 секунд') + # Задержка вывода из балансировщика входит в общий срок остановки, и на сами запросы + # должно остаться время. + if not 0 <= drain_delay <= shutdown_timeout - 2: + raise SystemExit('APP_DRAIN_DELAY должен быть от 0 до APP_SHUTDOWN_TIMEOUT - 2 секунд') + log_level = environ.get('APP_LOG_LEVEL', 'INFO').upper() + if log_level not in LOG_LEVELS: + raise SystemExit('APP_LOG_LEVEL должен быть одним из: ' + ', '.join(LOG_LEVELS)) + return Config( + host=environ.get('APP_HOST', '0.0.0.0'), + port=_integer(environ, 'APP_PORT', 8080, 1, 65535), + env=environ.get('APP_ENV', 'lab'), + log_level=log_level, + shutdown_timeout=shutdown_timeout, + drain_delay=drain_delay, + session_ttl=_integer(environ, 'SESSION_TTL_SECONDS', 3600, 1, MAX_SESSION_TTL), + idempotency_ttl=_integer(environ, 'IDEMPOTENCY_TTL_SECONDS', MIN_IDEMPOTENCY_TTL, + MIN_IDEMPOTENCY_TTL, MAX_SESSION_TTL), + db_pool_size=_integer(environ, 'DB_POOL_SIZE', 10, 1, 50), + database_url=read_secret('DATABASE_URL', environ), + cache_url=read_secret('CACHE_URL', environ), + # X-Instance-Id: в Docker hostname контейнера — короткий ID, по нему хук находит реплику. + instance_id=environ.get('INSTANCE_ID') or socket.gethostname(), + version=environ.get('APP_VERSION', 'dev'), + ) diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 0000000..dc0bb14 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,309 @@ +"""Доступ к PostgreSQL: ограниченный пул соединений, таймауты, только параметризованные запросы. + +Значения всегда передаются отдельно от текста SQL (%s), поэтому SQL-инъекция невозможна. +Каждый метод — одна транзакция: при любой ошибке внутри неё всё откатывается. +""" +import logging +import uuid +from contextlib import contextmanager + +import psycopg +from psycopg import errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .errors import ApiError, Unavailable, not_found +from .timeutil import format_time, from_micros, to_micros + +log = logging.getLogger('messenger') + +# Таймауты на стороне сервера БД: долгий запрос, ожидание блокировки и брошенная транзакция +# прерываются, а не держат соединение из пула бесконечно. +SESSION_OPTIONS = '-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=10000' + +# Отдельный клиентский предел: server-side таймеры не работают при docker pause. +DB_IO_TIMEOUT = 3.0 + + +class BoundedConnection(psycopg.Connection): + """Ограничивает сетевое ожидание, включая pool check, COMMIT и ROLLBACK. + + Connection.wait(timeout=...) поддерживается закреплённым psycopg 3.3.6. + После таймаута протокол может быть посреди ответа: такое соединение нельзя + возвращать в пул или ждать его ROLLBACK. Закрываем его в том же потоке; + pool заменит закрытое соединение. Ожидания server-side CANCEL здесь нет. + """ + + def wait(self, gen, interval=0.1, timeout=None): + limit = DB_IO_TIMEOUT if timeout is None else min(timeout, DB_IO_TIMEOUT) + try: + return super().wait(gen, interval=interval, timeout=limit) + except errors._WaitTimeout as error: + self.close() + raise psycopg.OperationalError('PostgreSQL client I/O timeout') from error + + +MEMBER_IDS = 'array(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids' + + +class UsernameTaken(Exception): + pass + + +class UnknownMember(Exception): + pass + + +class IdempotencyConflict(Exception): + """Тот же Idempotency-Key уже использован с другим телом запроса.""" + + +def create_pool(database_url, max_size): + return ConnectionPool( + database_url, + connection_class=BoundedConnection, + min_size=1, + max_size=max_size, # не больше max_size соединений с БД на процесс + timeout=3, # ждать свободное соединение не дольше 3 с, затем 503 + max_waiting=100, # очередь ожидающих запросов ограничена + check=ConnectionPool.check_connection, # оборванное соединение (рестарт БД) не выдаётся + kwargs={ + 'connect_timeout': 3, + 'application_name': 'messenger-api', + 'options': SESSION_OPTIONS, + 'row_factory': dict_row, + }, + name='messenger', + open=False, + ) + + +def _user(row): + return {'id': str(row['id']), 'username': row['username'], 'display_name': row['display_name'], + 'created_at': format_time(row['created_at'])} + + +def _chat(row): + return {'id': str(row['id']), 'title': row['title'], 'member_ids': [str(m) for m in row['member_ids']], + 'created_at': format_time(row['created_at'])} + + +def _message(row): + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']), + 'text': row['text'], 'attachment_ids': [str(x) for x in row['attachment_ids']], + 'created_at': format_time(row['created_at'])} + + +def attachment(row): + public_variant_fields = ('name', 'content_type', 'size_bytes', 'width', 'height', 'sha256') + return {**{k: str(row[k]) for k in ('id', 'chat_id', 'owner_id')}, + **{k: row[k] for k in ('filename', 'content_type', 'size_bytes', 'sha256')}, + 'status': row['status'], + 'variants': [{k: value[k] for k in public_variant_fields} for value in row.get('variants', [])], + 'error_code': row['error_code'], + 'created_at': format_time(row['created_at']), 'updated_at': format_time(row['updated_at'])} + + +def _page(rows, limit, convert): + """Из limit+1 строк: элементы страницы и ключ (created_at, id) последней, если есть продолжение.""" + items = rows[:limit] + has_more = len(rows) > limit + next_key = (to_micros(items[-1]['created_at']), str(items[-1]['id'])) if has_more else None + return [convert(row) for row in items], next_key + + +def _after(key): + return from_micros(key[0]), uuid.UUID(key[1]) + + +class Repository: + def __init__(self, pool): + self._pool = pool + + @contextmanager + def _transaction(self): + """Соединение из пула в транзакции: COMMIT при успехе, ROLLBACK при исключении.""" + try: + with self._pool.connection() as conn: + yield conn + except psycopg.OperationalError as error: + # Включая клиентский I/O timeout, когда процесс PostgreSQL приостановлен. + raise Unavailable('postgres') from error + + def ping(self): + with self._transaction() as conn: + conn.execute('SELECT 1') + + # --- пользователи ------------------------------------------------------- + + def create_user(self, username, display_name, password_hash): + try: + with self._transaction() as conn: + row = conn.execute( + 'INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) ' + 'RETURNING id, username, display_name, created_at', + (uuid.uuid4(), username, display_name, password_hash), + ).fetchone() + except errors.UniqueViolation: + # Уникальность username гарантирует индекс БД, поэтому гонка двух регистраций безопасна. + raise UsernameTaken + return _user(row) + + def get_user(self, user_id): + with self._transaction() as conn: + row = conn.execute( + 'SELECT id, username, display_name, created_at FROM users WHERE id = %s', (uuid.UUID(user_id),), + ).fetchone() + return _user(row) if row else None + + def get_credentials(self, username): + """(пользователь, хеш пароля) или (None, None).""" + with self._transaction() as conn: + row = conn.execute( + 'SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s', + (username,), + ).fetchone() + return (_user(row), row['password_hash']) if row else (None, None) + + # --- чаты --------------------------------------------------------------- + + def create_chat(self, title, member_ids): + """Чат и все участники одной транзакцией. + + Неизвестный участник нарушает внешний ключ chat_members.user_id; тогда транзакция + откатывается целиком и уже вставленная строка chats тоже исчезает. + """ + chat_id = uuid.uuid4() + members = [uuid.UUID(member_id) for member_id in member_ids] + try: + with self._transaction() as conn: + row = conn.execute( + 'INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at', (chat_id, title), + ).fetchone() + conn.execute( + 'INSERT INTO chat_members (chat_id, user_id, position) ' + 'SELECT %s, member.user_id, member.ordinality - 1 ' + 'FROM unnest(%s::uuid[]) WITH ORDINALITY AS member(user_id, ordinality)', + (chat_id, members), + ) + except errors.ForeignKeyViolation: + log.info('event=chat_rolled_back reason=unknown_member') + raise UnknownMember + return _chat(dict(row, member_ids=members)) + + def get_chat(self, chat_id, user_id): + """Чат, если он есть и user_id в нём состоит; иначе None.""" + with self._transaction() as conn: + row = conn.execute( + 'SELECT c.id, c.title, c.created_at, ' + MEMBER_IDS + ' FROM chats c ' + 'WHERE c.id = %s AND EXISTS (' + ' SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = %s)', + (uuid.UUID(chat_id), uuid.UUID(user_id)), + ).fetchone() + return _chat(row) if row else None + + def list_chats(self, user_id, after, limit): + select = ('SELECT c.id, c.title, c.created_at, ' + MEMBER_IDS + ' ' + 'FROM chat_members me JOIN chats c ON c.id = me.chat_id WHERE me.user_id = %s ') + order = 'ORDER BY c.created_at, c.id LIMIT %s' + with self._transaction() as conn: + if after is None: + rows = conn.execute(select + order, (uuid.UUID(user_id), limit + 1)).fetchall() + else: + rows = conn.execute(select + 'AND (c.created_at, c.id) > (%s, %s) ' + order, + (uuid.UUID(user_id), *_after(after), limit + 1)).fetchall() + return _page(rows, limit, _chat) + + # --- сообщения ---------------------------------------------------------- + + def add_message(self, chat_id, sender_id, text, key, request_hash, ttl_seconds, attachment_ids=()): + """Идемпотентная отправка: (сообщение, True если это повтор уже выполненного запроса). + + Строка чата блокируется до конца транзакции. Поэтому, во-первых, сообщения одного чата + записываются по очереди и created_at растёт в порядке фиксации: читатель, идущий по + cursor, не пропустит сообщение, зафиксированное позже уже прочитанного. Во-вторых, + одновременные запросы с одним ключом (область ключа включает путь чата) тоже идут по + очереди, даже через разные реплики: второй увидит ключ первого и вернёт то же сообщение. + Ключ и сообщение фиксируются одной транзакцией; при ошибке не остаётся ни того, ни другого. + """ + chat = uuid.UUID(chat_id) + scope = (uuid.UUID(sender_id), 'POST', '/api/v1/chats/%s/messages' % chat_id, key) + with self._transaction() as conn: + conn.execute('SELECT 1 FROM chats WHERE id = %s FOR NO KEY UPDATE', (chat,)) + stored = conn.execute( + 'SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, m.attachment_ids ' + 'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id ' + 'WHERE k.user_id = %s AND k.method = %s AND k.path = %s AND k.key = %s ' + ' AND k.created_at > clock_timestamp() - make_interval(secs => %s)', + scope + (ttl_seconds,), + ).fetchone() + if stored: + if stored['request_hash'] != request_hash: + raise IdempotencyConflict + return _message(stored), True + ids = [uuid.UUID(value) for value in attachment_ids] + found = conn.execute('SELECT id,status FROM attachments WHERE chat_id = %s AND id = ANY(%s::uuid[]) FOR SHARE', + (chat, ids)).fetchall() + if len(found) != len(ids): + raise not_found() + if any(value['status'] == 'failed' for value in found): + raise ApiError(409, 'conflict', 'Вложение не обработано') + row = conn.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids, created_at) ' + 'VALUES (%s, %s, %s, %s, %s, GREATEST(clock_timestamp(), ' + " (SELECT max(created_at) FROM messages WHERE chat_id = %s) + interval '1 microsecond')) " + 'RETURNING id, chat_id, sender_id, text, created_at, attachment_ids', + (uuid.uuid4(), chat, uuid.UUID(sender_id), text, ids, chat), + ).fetchone() + for aid in ids: + conn.execute('INSERT INTO message_attachments (message_id, attachment_id, chat_id) VALUES (%s,%s,%s)', + (row['id'], aid, chat)) + # Устаревшая запись с тем же ключом (старше срока хранения) заменяется новой. + conn.execute( + 'INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, message_id) ' + 'VALUES (%s, %s, %s, %s, %s, %s) ' + 'ON CONFLICT (user_id, method, path, key) DO UPDATE SET request_hash = EXCLUDED.request_hash, ' + ' message_id = EXCLUDED.message_id, created_at = EXCLUDED.created_at', + scope + (request_hash, row['id']), + ) + return _message(row), False + + def delete_expired_idempotency_keys(self, ttl_seconds): + """Удаляет ключи старше срока хранения; возвращает число удалённых.""" + with self._transaction() as conn: + return conn.execute( + 'DELETE FROM idempotency_keys WHERE created_at < clock_timestamp() - make_interval(secs => %s)', + (ttl_seconds,), + ).rowcount + + def list_messages(self, chat_id, after, limit): + """Страница истории. Запрос совпадает по форме с индексом messages_chat_history_idx.""" + select = 'SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id = %s ' + order = 'ORDER BY created_at, id LIMIT %s' + with self._transaction() as conn: + if after is None: + rows = conn.execute(select + order, (uuid.UUID(chat_id), limit + 1)).fetchall() + else: + rows = conn.execute(select + 'AND (created_at, id) > (%s, %s) ' + order, + (uuid.UUID(chat_id), *_after(after), limit + 1)).fetchall() + return _page(rows, limit, _message) + + def save_attachment(self, values, put): + from .jobs import Jobs + return Jobs(self).save_upload(values, put) + + save_upload = save_attachment + + def get_attachment(self, attachment_id, user_id): + with self._transaction() as conn: + row = conn.execute( + 'SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s', (uuid.UUID(attachment_id), uuid.UUID(user_id))).fetchone() + if row: + row['variants'] = conn.execute( + 'SELECT name,object_key,content_type,size_bytes,width,height,sha256 FROM attachment_variants ' + 'WHERE attachment_id=%s AND pipeline_version=%s ' + "ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END", + (row['id'], row['pipeline_version']), + ).fetchall() + return row diff --git a/src/messenger/dispatcher.py b/src/messenger/dispatcher.py new file mode 100644 index 0000000..2bb233e --- /dev/null +++ b/src/messenger/dispatcher.py @@ -0,0 +1,62 @@ +"""Transactional outbox publisher. A duplicate XADD after a crash is intentional.""" +import logging +import os +import signal +import socket +import sys +import threading +import uuid + +from .broker import Broker +from .config import read_secret +from .db import create_pool +from .jobs import Jobs + +log = logging.getLogger('messenger.dispatcher') + + +def dispatch_once(jobs, broker, worker): + published = 0 + for row in jobs.claim_outbox(worker=worker, limit=20, lease_seconds=15): + try: + # COMMIT of upload/outbox happened earlier. No HTTP request needs + # to wait for the broker; the durable DB intent is retried here. + broker.publish(row['payload']) + jobs.mark_published(row) + published += 1 + log.info('event=job_published job_id=%s attempt=%s', + row['payload']['job_id'], row['payload']['attempt']) + except Exception: + # If XADD succeeded but its response/DB mark was lost, republish. + # Releasing is an optimization; a dispatcher crash recovers by lease. + jobs.release_outbox(row, delay_seconds=1) + raise + return published + + +def main(): + logging.basicConfig(level=os.environ.get('APP_LOG_LEVEL', 'INFO'), + format='%(asctime)s %(levelname)s %(message)s', stream=sys.stdout) + stop = threading.Event() + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, lambda *_: stop.set()) + pool = create_pool(read_secret('DATABASE_URL'), 2) + pool.open(wait=True) + jobs, broker = Jobs(pool), Broker() + worker = 'dispatcher-%s-%s' % (socket.gethostname(), uuid.uuid4().hex[:8]) + try: + while not stop.is_set(): + try: + count = dispatch_once(jobs, broker, worker) + if not count: + stop.wait(0.5) + except Exception as error: + log.warning('event=outbox_publish_deferred error_type=%s', type(error).__name__) + stop.wait(1) + finally: + broker.close() + pool.close() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 0000000..f9e558e --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,29 @@ +"""Ошибки, которые HTTP-слой превращает в ответы контракта.""" + + +class ApiError(Exception): + """Ответ с ошибкой в формате {"error": {...}}; challenge — значение WWW-Authenticate для 401.""" + + def __init__(self, status, code, message, challenge=None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.challenge = challenge + + +class Unavailable(Exception): + """Хранилище (postgres или valkey) не ответило в срок; клиент получит 503.""" + + def __init__(self, dependency): + super().__init__(dependency) + self.dependency = dependency + + +def bad_request(message): + return ApiError(400, 'invalid_request', message) + + +def not_found(): + # Чужой объект неотличим от несуществующего. + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/healthcheck.py b/src/messenger/healthcheck.py new file mode 100644 index 0000000..934da12 --- /dev/null +++ b/src/messenger/healthcheck.py @@ -0,0 +1,26 @@ +"""Проверка для Docker HEALTHCHECK: python -m messenger.healthcheck. + +В slim-образе нет curl, поэтому готовность проверяется самим интерпретатором. +Код возврата 0 — контейнер healthy, 1 — unhealthy. +""" +import http.client +import os +import sys + + +def main(): + try: + connection = http.client.HTTPConnection('127.0.0.1', int(os.environ.get('APP_PORT', '8080')), timeout=2) + try: + connection.request('GET', '/health/ready', headers={'Connection': 'close'}) + response = connection.getresponse() + response.read() # дочитываем ответ, чтобы соединение закрылось штатно, а не обрывом + return 0 if response.status == 200 else 1 + finally: + connection.close() + except (OSError, ValueError, http.client.HTTPException): + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/image_check.py b/src/messenger/image_check.py new file mode 100644 index 0000000..8df13fb --- /dev/null +++ b/src/messenger/image_check.py @@ -0,0 +1,29 @@ +"""Декодирование в отдельном процессе: RAM освобождается при выходе, время ограничено родителем.""" +import sys +import warnings + +from PIL import Image, UnidentifiedImageError + + +def main(): + path, expected, pixels = sys.argv[1:] + Image.MAX_IMAGE_PIXELS = int(pixels) + warnings.simplefilter('error', Image.DecompressionBombWarning) + try: + with Image.open(path, formats=['PNG', 'JPEG']) as image: + if image.width * image.height > int(pixels): + return 13 + if Image.MIME[image.format] != expected: + return 15 + image.verify() + with Image.open(path, formats=['PNG', 'JPEG']) as image: + image.load() # verify одного JPEG недостаточно: проверяем и распаковку. + return 0 + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + return 13 + except (OSError, ValueError, SyntaxError, UnidentifiedImageError): + return 15 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/image_process.py b/src/messenger/image_process.py new file mode 100644 index 0000000..00abbfb --- /dev/null +++ b/src/messenger/image_process.py @@ -0,0 +1,123 @@ +"""Bounded image decoder, run as a short lived subprocess by the worker. + +The source is never rewritten. All generated images have EXIF/ICC/XMP removed. +JPEG quality is 85; WebP quality is 80 (method 4). Alpha is flattened on white. +""" +import hashlib +import json +import os +import sys +import warnings + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +MAX_OUTPUT_BYTES = 32 * 1024 * 1024 + + +class InvalidImage(Exception): + def __init__(self, code='invalid_image'): + self.code = code + super().__init__(code) + + +def fit_dimensions(width, height, bound): + """Floor the proportional dimensions, without upscaling or zero pixels.""" + if max(width, height) <= bound: + return width, height + if width >= height: + return bound, max(1, height * bound // width) + return max(1, width * bound // height), bound + + +def _metadata(path, name, content_type, dimensions): + with open(path, 'rb') as output: + digest = hashlib.file_digest(output, 'sha256').hexdigest() + return dict(name=name, content_type=content_type, size_bytes=os.path.getsize(path), + sha256=digest, width=dimensions[0], height=dimensions[1]) + + +def process_image(source_path, output_dir, expected_type, max_pixels=MAX_PIXELS): + # Import after resource limits have been set in main(). The long lived worker + # never imports Pillow or holds a fully decoded untrusted image in memory. + from PIL import Image, ImageOps, UnidentifiedImageError + + Image.MAX_IMAGE_PIXELS = max_pixels + warnings.simplefilter('error', Image.DecompressionBombWarning) + if os.path.getsize(source_path) > MAX_BYTES: + raise InvalidImage('image_too_large') + try: + with Image.open(source_path, formats=['PNG', 'JPEG']) as source: + if source.width * source.height > max_pixels: + raise InvalidImage('image_too_large') + if Image.MIME[source.format] != expected_type: + raise InvalidImage() + source.load() + ImageOps.exif_transpose(source, in_place=True) + if source.mode == 'RGBA' or source.mode == 'LA' or 'transparency' in source.info: + rgba = source.convert('RGBA') + rgb = Image.new('RGB', source.size, 'white') + alpha = rgba.getchannel('A') + rgb.paste(rgba, mask=alpha) + alpha.close() + rgba.close() + else: + rgb = source.convert('RGB') + try: + metadata = [] + for name, bound, filename, fmt, mime in ( + ('thumbnail', 256, 'thumbnail.jpg', 'JPEG', 'image/jpeg'), + ('crop', 128, 'crop.jpg', 'JPEG', 'image/jpeg'), + ('optimized', 1280, 'optimized.webp', 'WEBP', 'image/webp'), + ): + if name == 'crop': + # Crop only the central square before resizing, avoiding a + # large upscaled intermediate for extremely narrow images. + side = min(rgb.size) + left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 + square = rgb.crop((left, top, left + side, top + side)) + variant = square.resize((128, 128), Image.Resampling.LANCZOS) + square.close() + else: + variant = rgb.resize(fit_dimensions(*rgb.size, bound), Image.Resampling.LANCZOS) + try: + path = os.path.join(output_dir, filename) + options = dict(quality=80, method=4) if fmt == 'WEBP' else dict(quality=85, optimize=True) + variant.save(path, format=fmt, exif=b'', icc_profile=None, **options) + metadata.append(_metadata(path, name, mime, variant.size)) + finally: + variant.close() + return metadata + finally: + rgb.close() + except (Image.DecompressionBombWarning, Image.DecompressionBombError, MemoryError): + raise InvalidImage('image_too_large') from None + except (OSError, ValueError, SyntaxError, UnidentifiedImageError): + raise InvalidImage() from None + + +def set_resource_limits(memory_mib, timeout_seconds): + if sys.platform == 'win32': + return # Production containers are Linux; tests also run on Windows. + import resource + memory = memory_mib * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_FSIZE, (MAX_OUTPUT_BYTES, MAX_OUTPUT_BYTES)) + cpu = max(1, int(timeout_seconds) + 1) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def main(): + source, output_dir, expected, memory, timeout = sys.argv[1:] + set_resource_limits(int(memory), float(timeout)) + try: + variants = process_image(source, output_dir, expected) + except InvalidImage as error: + return 13 if error.code == 'image_too_large' else 15 + with open(os.path.join(output_dir, 'result.json'), 'w', encoding='utf-8') as output: + json.dump(variants, output, separators=(',', ':')) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/jobs.py b/src/messenger/jobs.py new file mode 100644 index 0000000..35f498c --- /dev/null +++ b/src/messenger/jobs.py @@ -0,0 +1,319 @@ +"""Durable attachment state machine, fenced leases and transactional outbox. + +No broker call or image processing runs inside these transactions. A worker may +repeat deterministic S3 writes, but only the current, unexpired lease can publish +their metadata. Every retry has an outbox row in the same commit as its state. +""" +import re +import uuid + +from psycopg.types.json import Jsonb + +from .db import Repository, attachment +from .timeutil import format_time + +VARIANT_NAMES = frozenset(('thumbnail', 'crop', 'optimized')) +FAULT_ACTIONS = frozenset(('crash_after_objects', 'crash_after_ready', 'transient', 'permanent', 'stall_processing')) + + +class _LeaseLost(Exception): + """Roll back writes when the deadline passes during the result transaction.""" + + +def _uuid(value): + return value if isinstance(value, uuid.UUID) else uuid.UUID(str(value)) + + +def _safe_error(value): + return value if isinstance(value, str) and re.fullmatch(r'[a-z][a-z0-9_]{0,63}', value) else 'processing_failed' + + +class Jobs: + def __init__(self, repo_or_pool): + self.repo = repo_or_pool if isinstance(repo_or_pool, Repository) else Repository(repo_or_pool) + + @staticmethod + def _load(conn, job_id, lock=False): + return conn.execute( + 'SELECT j.*, j.id AS job_id, a.staging_key, a.object_key, a.filename, a.content_type, ' + 'a.size_bytes, a.sha256, a.pipeline_version AS attachment_version ' + 'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s' + + (' FOR UPDATE OF j' if lock else ''), (_uuid(job_id),), + ).fetchone() + + @staticmethod + def _enqueue(conn, row): + conn.execute( + 'INSERT INTO attachment_outbox (id,job_id,attempt,available_at) VALUES (%s,%s,%s,%s) ' + 'ON CONFLICT (job_id,attempt) DO NOTHING', + (uuid.uuid4(), row['job_id'], row['attempt'] + 1, row['available_at']), + ) + + @staticmethod + def _audit(conn, job_id, action, actor, reason, details=None): + conn.execute('INSERT INTO job_audit (job_id,action,actor,reason,details) VALUES (%s,%s,%s,%s,%s)', + (_uuid(job_id), action, actor, reason, Jsonb(details or {}))) + + def save_upload(self, values, put): + values = dict(values) + values['staging_key'] = values.get('staging_key', values['object_key']) + with self.repo._transaction() as conn: + # Shared with orphan cleanup: a successful PUT awaiting COMMIT is live. + conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (values['staging_key'],)) + put() + row = conn.execute( + 'INSERT INTO attachments (id,chat_id,owner_id,object_key,staging_key,filename,content_type,size_bytes,sha256) ' + 'VALUES (%(id)s,%(chat_id)s,%(owner_id)s,%(object_key)s,%(staging_key)s,%(filename)s,%(content_type)s,%(size_bytes)s,%(sha256)s) ' + 'RETURNING *', values, + ).fetchone() + job = conn.execute( + 'INSERT INTO attachment_jobs (id,attachment_id) VALUES (%s,%s) RETURNING *, id AS job_id', + (uuid.uuid4(), row['id']), + ).fetchone() + self._enqueue(conn, job) + return attachment(row) + + def claim(self, job_id, worker, lease_seconds, max_attempts): + if lease_seconds <= 0 or max_attempts < 1 or not worker: + raise ValueError('invalid lease configuration') + with self.repo._transaction() as conn: + row = self._load(conn, job_id, lock=True) + if not row or row['status'] != 'queued' or row['pipeline_version'] != row['attachment_version']: + return None + due = conn.execute('SELECT %s::timestamptz <= clock_timestamp() AS due', (row['available_at'],)).fetchone()['due'] + if not due: + return None + if row['run_attempts'] >= max_attempts: + self._transition(conn, row, 'failed', 'retry_exhausted', 0) + return None + lease_token = uuid.uuid4() + updated = conn.execute( + "UPDATE attachment_jobs SET status='processing',attempt=attempt+1,run_attempts=run_attempts+1," + 'worker_id=%s,lease_token=%s,lease_until=clock_timestamp()+make_interval(secs=>%s), ' + 'updated_at=clock_timestamp() WHERE id=%s RETURNING *', + (worker, lease_token, lease_seconds, row['id']), + ).fetchone() + conn.execute("UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (row['attachment_id'],)) + return dict(row, **updated, job_id=updated['id']) + + @staticmethod + def _current(conn, claim): + row = conn.execute( + "SELECT j.*,j.id AS job_id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s AND j.status='processing' AND j.lease_token=%s AND j.lease_until>clock_timestamp() " + 'AND j.attempt=%s AND j.pipeline_version=%s AND a.pipeline_version=j.pipeline_version ' + 'FOR UPDATE OF j,a', + (_uuid(claim['job_id']), _uuid(claim['lease_token']), claim['attempt'], claim['pipeline_version']), + ).fetchone() + # A row lock can have waited. Recheck the wall clock after both locks are held. + if row and not conn.execute('SELECT %s::timestamptz>clock_timestamp() AS valid', + (row['lease_until'],)).fetchone()['valid']: + return None + return row + + def finish(self, claim, original_object_key, variants): + if len(variants) != len(VARIANT_NAMES) or {v['name'] for v in variants} != VARIANT_NAMES: + raise ValueError('all three grade-4 variants are required') + try: + with self.repo._transaction() as conn: + row = self._current(conn, claim) + if not row: + return False + for variant in variants: + conn.execute( + 'INSERT INTO attachment_variants (attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (attachment_id,pipeline_version,name) DO UPDATE SET ' + 'object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,size_bytes=EXCLUDED.size_bytes,' + 'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256', + (row['attachment_id'], row['pipeline_version'], variant['name'], variant['object_key'], + variant['content_type'], variant['size_bytes'], variant['width'], variant['height'], variant['sha256']), + ) + conn.execute( + "UPDATE attachments SET status='ready',object_key=%s,error_code=NULL,updated_at=clock_timestamp()," + 'staging_cleanup_pending=(staging_key<>%s AND staging_deleted_at IS NULL) WHERE id=%s', + (original_object_key, original_object_key, row['attachment_id']), + ) + fenced = conn.execute( + "UPDATE attachment_jobs SET status='ready',error_code=NULL,worker_id=NULL,lease_token=NULL,lease_until=NULL," + 'updated_at=clock_timestamp() WHERE id=%s AND lease_token=%s AND lease_until>clock_timestamp()', + (row['id'], row['lease_token']), + ).rowcount + if fenced != 1: + raise _LeaseLost + return True + except _LeaseLost: + return False + + def _transition(self, conn, row, status, error_code, delay_seconds): + updated = conn.execute( + 'UPDATE attachment_jobs SET status=%s,error_code=%s,worker_id=NULL,lease_token=NULL,lease_until=NULL,' + 'available_at=clock_timestamp()+make_interval(secs=>%s),updated_at=clock_timestamp() ' + 'WHERE id=%s RETURNING *,id AS job_id', + (status, error_code, max(0, delay_seconds), row['id']), + ).fetchone() + conn.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s', + (status, error_code if status == 'failed' else None, row['attachment_id'])) + if status == 'queued': + self._enqueue(conn, updated) + if error_code: + self._audit(conn, row['id'], 'retry', row.get('worker_id') or 'recovery', error_code, + {'attempt': row['attempt'], 'run_attempts': row['run_attempts'], + 'delay_seconds': max(0, delay_seconds), 'available_at': format_time(updated['available_at'])}) + else: + self._audit(conn, row['id'], 'failed', row.get('worker_id') or 'recovery', error_code, + {'attempt': row['attempt'], 'run_attempts': row['run_attempts'], 'error_code': error_code}) + + def fail(self, claim, error_code, permanent, max_attempts, delay_seconds): + with self.repo._transaction() as conn: + row = self._current(conn, claim) + if not row: + return False + exhausted = row['run_attempts'] >= max_attempts + status = 'failed' if permanent or exhausted else 'queued' + code = 'retry_exhausted' if exhausted and not permanent else _safe_error(error_code) + self._transition(conn, row, status, code, delay_seconds) + return True + + def recover_expired(self, max_attempts, limit=100): + with self.repo._transaction() as conn: + rows = conn.execute( + "SELECT *,id AS job_id FROM attachment_jobs WHERE status='processing' AND lease_until<=clock_timestamp() " + 'ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED', (limit,), + ).fetchall() + for row in rows: + exhausted = row['run_attempts'] >= max_attempts + self._transition(conn, row, 'failed' if exhausted else 'queued', + 'retry_exhausted' if exhausted else 'lease_expired', 0) + return len(rows) + + def claim_outbox(self, worker='dispatcher', limit=20, lease_seconds=15): + token = uuid.uuid4() + with self.repo._transaction() as conn: + rows = conn.execute( + 'WITH due AS (SELECT o.id FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id ' + "WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.status='queued' " + 'AND o.attempt=j.attempt+1 AND (o.lease_until IS NULL OR o.lease_until<=clock_timestamp()) ' + 'ORDER BY o.available_at,o.created_at LIMIT %s FOR UPDATE OF o SKIP LOCKED) ' + 'UPDATE attachment_outbox o SET worker_id=%s,lease_token=%s,' + 'lease_until=clock_timestamp()+make_interval(secs=>%s) FROM due WHERE o.id=due.id ' + 'RETURNING o.*', (limit, worker, token, lease_seconds), + ).fetchall() + for row in rows: + job = self._load(conn, row['job_id']) + row['payload'] = {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], + 'attempt': row['attempt'], 'created_at': format_time(row['created_at'])} + return rows + + def mark_published(self, outbox_row): + with self.repo._transaction() as conn: + return conn.execute( + 'UPDATE attachment_outbox SET published_at=clock_timestamp(),worker_id=NULL,lease_token=NULL,lease_until=NULL ' + 'WHERE id=%s AND lease_token=%s AND published_at IS NULL AND lease_until>clock_timestamp()', + (_uuid(outbox_row['id']), _uuid(outbox_row['lease_token'])), + ).rowcount == 1 + + def release_outbox(self, outbox_row, delay_seconds=1): + with self.repo._transaction() as conn: + return conn.execute( + 'UPDATE attachment_outbox SET worker_id=NULL,lease_token=NULL,lease_until=NULL,' + 'available_at=clock_timestamp()+make_interval(secs=>%s) WHERE id=%s AND lease_token=%s AND published_at IS NULL', + (max(0, delay_seconds), _uuid(outbox_row['id']), _uuid(outbox_row['lease_token'])), + ).rowcount == 1 + + def pending_cleanup(self, limit=100): + with self.repo._transaction() as conn: + return conn.execute( + "SELECT id AS attachment_id,staging_key FROM attachments WHERE status='ready' AND staging_cleanup_pending " + 'ORDER BY updated_at LIMIT %s', (limit,), + ).fetchall() + + def mark_staging_deleted(self, attachment_id, staging_key): + with self.repo._transaction() as conn: + return conn.execute( + "UPDATE attachments SET staging_cleanup_pending=false,staging_deleted_at=clock_timestamp() " + "WHERE id=%s AND staging_key=%s AND status='ready' AND staging_cleanup_pending", + (_uuid(attachment_id), staging_key), + ).rowcount == 1 + + def replay_failed(self, attachment_id, reason, actor='operator'): + if not reason or not reason.strip() or len(reason) > 500 or not actor or len(actor) > 120: + raise ValueError('replay requires an actor and a reason (1-500 characters)') + with self.repo._transaction() as conn: + row = conn.execute('SELECT *,id AS job_id FROM attachment_jobs WHERE attachment_id=%s FOR UPDATE', + (_uuid(attachment_id),)).fetchone() + if not row: + raise ValueError('attachment job not found') + if row['status'] == 'ready': + self._audit(conn, row['id'], 'replay_noop', actor, reason, {'attempt': row['attempt']}) + return self._summary(row) + if row['status'] != 'failed': + raise ValueError('only failed jobs may be replayed') + original = conn.execute('SELECT staging_deleted_at FROM attachments WHERE id=%s FOR UPDATE', + (row['attachment_id'],)).fetchone() + if original['staging_deleted_at'] is not None: + raise ValueError('staging_expired') + conn.execute('UPDATE attachment_jobs SET run_attempts=0 WHERE id=%s', (row['id'],)) + self._transition(conn, row, 'queued', None, 0) + self._audit(conn, row['id'], 'replay', actor, reason, {'next_attempt': row['attempt'] + 1}) + return dict(self._summary(row), status='queued', run_attempts=0, error_code=None) + + @staticmethod + def _summary(row): + return {'job_id': str(row['id']), 'attachment_id': str(row['attachment_id']), + 'status': row['status'], 'attempt': row['attempt'], 'run_attempts': row['run_attempts'], + 'error_code': row['error_code']} + + def failed(self, limit=100): + with self.repo._transaction() as conn: + return [self._summary(row) for row in conn.execute( + "SELECT * FROM attachment_jobs WHERE status='failed' ORDER BY updated_at LIMIT %s", (limit,), + ).fetchall()] + + def inspect(self, job_id): + with self.repo._transaction() as conn: + row = self._load(conn, job_id) + return dict(self._summary(row), pipeline_version=row['pipeline_version'], + available_at=format_time(row['available_at']), + lease_until=format_time(row['lease_until']) if row['lease_until'] else None) if row else None + + def delivery_state(self, job_id): + with self.repo._transaction() as conn: + row = self._load(conn, job_id) + if not row: + return None + count = conn.execute('SELECT count(*) AS n FROM attachment_variants ' + 'WHERE attachment_id=%s AND pipeline_version=%s', + (row['attachment_id'], row['pipeline_version'])).fetchone()['n'] + return dict(self._summary(row), pipeline_version=row['pipeline_version'], + created_at=format_time(row['created_at']), available_at=format_time(row['available_at']), + lease_until=format_time(row['lease_until']) if row['lease_until'] else None, + variant_count=count) + + def inject_fault(self, job_id, action, count, actor, reason): + if action not in FAULT_ACTIONS or not isinstance(count, int) or not 0 <= count <= 100: + raise ValueError('invalid fault action/count') + if not actor or len(actor) > 120 or not reason or not reason.strip() or len(reason) > 500: + raise ValueError('fault injection requires an actor and a reason') + with self.repo._transaction() as conn: + if not self._load(conn, job_id, lock=True): + raise ValueError('job not found') + conn.execute('INSERT INTO job_faults (job_id,action,remaining) VALUES (%s,%s,%s) ' + 'ON CONFLICT (job_id,action) DO UPDATE SET remaining=EXCLUDED.remaining', + (_uuid(job_id), action, count)) + self._audit(conn, job_id, 'fault_configured', actor, reason, {'action': action, 'count': count}) + + def consume_fault(self, job_id, action): + if action not in FAULT_ACTIONS: + raise ValueError('invalid fault action') + with self.repo._transaction() as conn: + row = conn.execute('UPDATE job_faults SET remaining=remaining-1 ' + 'WHERE job_id=%s AND action=%s AND remaining>0 RETURNING remaining', + (_uuid(job_id), action)).fetchone() + if not row: + return False + self._audit(conn, job_id, 'fault_consumed', 'worker', 'explicit operator fault', + {'action': action, 'remaining': row['remaining']}) + return True diff --git a/src/messenger/jobs_cli.py b/src/messenger/jobs_cli.py new file mode 100644 index 0000000..aab5c16 --- /dev/null +++ b/src/messenger/jobs_cli.py @@ -0,0 +1,74 @@ +"""Explicit operator tools: inspected failed registry, audited replay and fault tests.""" +import argparse +import json + +from .broker import Broker +from .config import read_secret +from .db import create_pool +from .jobs import FAULT_ACTIONS, Jobs + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + failed = commands.add_parser('failed', help='list terminal failures without storage secrets') + failed.add_argument('--limit', type=int, default=100) + show = commands.add_parser('show', help='inspect a job by job ID') + show.add_argument('job_id') + replay = commands.add_parser('replay', help='replay a failed attachment; ready is an audited no-op') + replay.add_argument('attachment_id') + replay.add_argument('--actor', required=True) + replay.add_argument('--reason', required=True) + inject = commands.add_parser('inject', help='configure an audited, persistent, job-specific fault') + inject.add_argument('job_id') + inject.add_argument('--action', choices=sorted(FAULT_ACTIONS), required=True) + inject.add_argument('--count', type=int, default=1, help='number of invocations; 0 disables the hook') + inject.add_argument('--actor', required=True) + inject.add_argument('--reason', required=True) + duplicate = commands.add_parser('redeliver', help='publish a duplicate delivery; ready work stays ready') + duplicate.add_argument('job_id') + commands.add_parser('cleanup', help='retry deletion of staging for durably ready attachments') + args = parser.parse_args() + if args.command == 'failed' and not 1 <= args.limit <= 1000: + parser.error('limit must be in 1..1000') + pool = create_pool(read_secret('DATABASE_URL'), 1) + pool.open(wait=True) + jobs = Jobs(pool) + try: + if args.command == 'failed': + result = jobs.failed(args.limit) + elif args.command == 'show': + result = jobs.delivery_state(args.job_id) + if result is None: + parser.error('job not found') + elif args.command == 'replay': + result = jobs.replay_failed(args.attachment_id, args.reason, actor=args.actor) + elif args.command == 'inject': + jobs.inject_fault(args.job_id, args.action, args.count, args.actor, args.reason) + result = dict(job_id=args.job_id, action=args.action, count=args.count, audit='fault_configured') + elif args.command == 'redeliver': + state = jobs.delivery_state(args.job_id) + if state is None: + parser.error('job not found') + payload = dict(schema_version=1, job_id=state['job_id'], type='attachment.process.v1', + attachment_id=state['attachment_id'], pipeline_version=state['pipeline_version'], + attempt=max(1, state['attempt']), created_at=state['created_at']) + broker = Broker() + try: + message_id = broker.publish(payload) + finally: + broker.close() + result = dict(job_id=state['job_id'], message_id=message_id, status=state['status']) + else: + from .objects import ObjectStore + from .worker import cleanup_ready + result = {'staging_deleted': cleanup_ready(jobs, ObjectStore())} + except ValueError as error: + parser.error(str(error)) + finally: + pool.close() + print(json.dumps(result, separators=(',', ':'), sort_keys=True)) + + +if __name__ == '__main__': + main() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 0000000..8884080 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,77 @@ +"""Миграции схемы: python -m messenger.migrate. + +Файлы migrations/NNNN_*.sql применяются по порядку номеров, каждый в своей транзакции. +Применённые версии и контрольные суммы записываются в schema_migrations, поэтому повторный +запуск ничего не дублирует. Изменение уже применённого файла останавливает запуск: менять +схему нужно новым файлом. Advisory lock не даёт двум процессам применять миграции одновременно. +""" +import hashlib +import logging +import sys +import time +from pathlib import Path + +import psycopg + +from .config import read_secret + +log = logging.getLogger('messenger.migrate') + +MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / 'migrations' +LOCK_ID = 2026_0002 # произвольная константа для pg_advisory_lock + +CREATE_TABLE = ''' +CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + checksum text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +)''' + + +def _read(path): + # Окончания строк нормализуются: checkout в Windows (CRLF) не должен менять контрольную сумму. + sql = path.read_text(encoding='utf-8').replace('\r\n', '\n') + return sql, hashlib.sha256(sql.encode('utf-8')).hexdigest() + + +def migrate(database_url, directory=MIGRATIONS_DIR): + files = sorted(directory.glob('[0-9][0-9][0-9][0-9]_*.sql')) + if not files: + raise SystemExit('Нет файлов миграций в ' + str(directory)) + applied_now = 0 + with psycopg.connect(database_url, autocommit=True, connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_lock(%s)', (LOCK_ID,)) + try: + conn.execute(CREATE_TABLE) + applied = dict(conn.execute('SELECT version, checksum FROM schema_migrations').fetchall()) + for path in files: + version = path.stem + sql, checksum = _read(path) + if version in applied: + if applied[version] != checksum: + raise SystemExit('Миграция %s изменена после применения: добавьте новую вместо правки' % version) + continue + started = time.monotonic() + with conn.transaction(): + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)', (version, checksum)) + applied_now += 1 + log.info('event=migration_applied version=%s duration_ms=%.0f', version, (time.monotonic() - started) * 1000) + finally: + conn.execute('SELECT pg_advisory_unlock(%s)', (LOCK_ID,)) + log.info('event=migrations_done applied_now=%d total=%d', applied_now, len(files)) + + +def main(): + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(levelname)s %(message)s') + try: + migrate(read_secret('DATABASE_URL')) + except psycopg.Error as error: + # Сообщения psycopg не содержат пароль, а первая строка объясняет причину. + log.error('event=migration_failed error=%s detail=%s', type(error).__name__, str(error).splitlines()[0][:300]) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/objects.py b/src/messenger/objects.py new file mode 100644 index 0000000..6db4194 --- /dev/null +++ b/src/messenger/objects.py @@ -0,0 +1,40 @@ +"""Приватный S3; ограниченные таймауты и отсутствие скрытых повторов PUT.""" +import os + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import read_secret +from .errors import Unavailable + + +class ObjectStore: + def __init__(self, admin=False): + self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + prefix = 'S3_ADMIN_' if admin else 'S3_' + self.client = boto3.client( + 's3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.environ.get('S3_REGION', 'us-east-1'), + aws_access_key_id=read_secret(prefix + 'ACCESS_KEY_ID'), + aws_secret_access_key=read_secret(prefix + 'SECRET_ACCESS_KEY'), + config=Config(connect_timeout=1, read_timeout=3, max_pool_connections=16, + retries={'total_max_attempts': 1}, s3={'addressing_style': 'path'}, + request_checksum_calculation='when_required', + response_checksum_validation='when_required')) + + def call(self, method, **kwargs): + try: + return getattr(self.client, method)(Bucket=self.bucket, **kwargs) + except (BotoCoreError, ClientError, OSError) as error: + raise Unavailable('s3') from error + + def ping(self): + self.call('head_bucket') + + def put(self, key, file, size, content_type): + file.seek(0) + self.call('put_object', Key=key, Body=file, ContentLength=size, ContentType=content_type) + + def get(self, key): + return self.call('get_object', Key=key)['Body'] diff --git a/src/messenger/passwords.py b/src/messenger/passwords.py new file mode 100644 index 0000000..51546ba --- /dev/null +++ b/src/messenger/passwords.py @@ -0,0 +1,50 @@ +"""Хеширование паролей: Argon2id из библиотеки argon2-cffi. + +Хеш хранит алгоритм, параметры и соль в одной строке вида $argon2id$v=19$m=...,t=...,p=...$соль$хеш. +""" +import secrets +import threading +from contextlib import contextmanager + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +from .errors import Unavailable + +# Минимальные параметры из OWASP Password Storage Cheat Sheet: 19 МиБ памяти, 2 прохода, 1 поток. +_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1) + +# Каждый расчёт занимает 19 МиБ памяти. Одновременно считаем не больше 4 хешей на процесс, +# чтобы пик памяти реплики был предсказуем при лимите контейнера (compose.yaml, 256 МБ). +# Остальные запросы ждут очереди; дольше 5 с — ответ 503 вместо бесконечного ожидания. +MAX_CONCURRENT_HASHES = 4 +_slots = threading.BoundedSemaphore(MAX_CONCURRENT_HASHES) + +# Хеш случайного пароля. Проверка против него для несуществующего username занимает +# столько же времени, сколько обычная: по задержке ответа нельзя узнать, есть ли пользователь. +_DUMMY_HASH = _hasher.hash(secrets.token_hex(16)) + + +@contextmanager +def _slot(): + if not _slots.acquire(timeout=5): + raise Unavailable('password-hashing') + try: + yield + finally: + _slots.release() + + +def hash_password(password): + with _slot(): + return _hasher.hash(password) + + +def verify_password(password_hash, password): + """True, если пароль подходит к хешу; при password_hash=None всегда False за то же время.""" + with _slot(): + try: + _hasher.verify(password_hash or _DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None diff --git a/src/messenger/server.py b/src/messenger/server.py new file mode 100644 index 0000000..4ec2f94 --- /dev/null +++ b/src/messenger/server.py @@ -0,0 +1,498 @@ +"""HTTP-слой: маршрутизация, аутентификация, ответы в формате контракта, логи запросов.""" +import json +import hashlib +import logging +import os +import re +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs + +from . import validation +from .db import IdempotencyConflict, UnknownMember, UsernameTaken +from .errors import ApiError, Unavailable, bad_request, not_found +from .passwords import hash_password, verify_password +from .timeutil import format_time +from .db import attachment + +log = logging.getLogger('messenger') + +# Сообщение в 4000 символов даже в виде \uXXXX-последовательностей меньше 64 КиБ. +MAX_BODY_BYTES = 64 * 1024 +# Тело до этого размера дочитывается ради аккуратного 413; больше — соединение просто закрывается. +MAX_DISCARD_BYTES = 1024 * 1024 + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + +CHAT = r'/api/v1/chats/(?P[^/]+)' +ROUTES = tuple((method, re.compile(pattern), handler) for method, pattern, handler in ( + ('GET', r'/health/live', '_live'), + ('GET', r'/health/ready', '_ready'), + ('POST', r'/api/v1/users', '_create_user'), + ('GET', r'/api/v1/users/me', '_current_user'), + ('POST', r'/api/v1/auth/sessions', '_create_session'), + ('DELETE', r'/api/v1/auth/sessions/current', '_delete_session'), + ('POST', r'/api/v1/chats', '_create_chat'), + ('GET', r'/api/v1/chats', '_list_chats'), + ('GET', CHAT, '_get_chat'), + ('POST', CHAT + r'/messages', '_create_message'), + ('GET', CHAT + r'/messages', '_list_messages'), + ('POST', CHAT + r'/attachments', '_upload'), + ('GET', r'/api/v1/attachments/(?P[^/]+)', '_attachment'), + ('GET', r'/api/v1/attachments/(?P[^/]+)/content', '_content'), +)) + + +class Download: + def __init__(self, stream, row): + self.stream, self.row = stream, row + + +class MessengerServer(ThreadingHTTPServer): + """Поток на соединение; считает выполняющиеся запросы для плавной остановки.""" + + # Потоки-демоны не держат процесс: срок остановки ограничивает wait_idle(). + daemon_threads = True + # Очередь соединений по умолчанию (5) мала для десятков одновременных клиентов. + request_queue_size = 128 + + def __init__(self, address, repo, sessions, instance_id, idempotency_ttl, objects): + super().__init__(address, RequestHandler) + self.repo = repo + self.sessions = sessions + self.instance_id = instance_id + self.idempotency_ttl = idempotency_ttl + self.objects = objects + # Один декодер на реплику: 40M RGBA <=160 MB + память интерпретаторов. + self.upload_slots = threading.BoundedSemaphore(1) + self.max_upload = int(os.environ.get('MAX_UPLOAD_BYTES', '10485760')) + self.max_pixels = int(os.environ.get('MAX_IMAGE_PIXELS', '40000000')) + if not 1 <= self.max_upload <= 10485760 or not 1 <= self.max_pixels <= 40000000: + raise ValueError('Некорректный лимит вложений') + self._state = threading.Condition() + self._active = 0 + self._draining = False + self.ready = True + + def begin_not_ready(self): + """Первый шаг остановки: /health/ready отвечает 503, остальные запросы ещё обслуживаются. + + Балансировщик видит это при следующих проверках и перестаёт присылать сюда запросы. + """ + self.ready = False + + def try_begin_request(self): + """False, если сервер уже останавливается и запрос принимать нельзя.""" + with self._state: + if self._draining: + return False + self._active += 1 + return True + + def end_request(self): + with self._state: + self._active -= 1 + self._state.notify_all() + + def begin_drain(self): + with self._state: + self._draining = True + + def wait_idle(self, timeout): + """Ждёт завершения начатых запросов; True, если успели за timeout секунд.""" + with self._state: + return self._state.wait_for(lambda: self._active == 0, timeout) + + def handle_error(self, request, client_address): + # Обрыв соединения клиентом — обычная ситуация, а не ошибка сервера: без traceback. + error = sys.exc_info()[0] + level = logging.DEBUG if issubclass(error, ConnectionError) else logging.WARNING + log.log(level, 'event=connection_error error=%s', error.__name__) + + +class RequestHandler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + # Секунды ожидания данных от клиента: зависшее соединение не занимает поток вечно. + timeout = 15 + + def do_GET(self): + self._handle() + + do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = do_GET + + # --- общий путь запроса ------------------------------------------------- + + def _handle(self): + started = time.monotonic() + self.request_id = uuid.uuid4().hex + self.user_id = '-' + path, _, query = self.path.partition('?') + status = 0 # 0 в логе: ответ отправить не удалось + accepted = self.server.try_begin_request() + try: + if accepted: + status, payload, headers = self._process(path, query) + else: + self.close_connection = True + status, payload, headers = self._error(ApiError(503, 'unavailable', 'Сервер завершает работу')) + self._send(status, payload, headers) + except OSError: + status = 0 + self.close_connection = True + finally: + if accepted: + self.server.end_request() + # В лог попадают только метаданные: тела запросов, тексты сообщений, пароли и токены не пишутся. + log.log( + logging.DEBUG if path.startswith('/health/') else logging.INFO, + 'request_id=%s method=%s path=%s status=%d duration_ms=%.1f user_id=%s', + self.request_id, self.command, re.sub(r'[^\x21-\x7e]', '?', path)[:200], status, + (time.monotonic() - started) * 1000, self.user_id, + ) + + def _process(self, path, query): + """(статус, тело или None, дополнительные заголовки).""" + try: + upload = self.command == 'POST' and re.fullmatch(CHAT + r'/attachments', path) + if upload: + # До чтения файла проверяем токен и чат. При ошибке непрочитанное тело + # не должно интерпретироваться как следующий запрос keep-alive. + self.close_connection = True + body = b'' if upload else self._read_body() + for method, pattern, handler in ROUTES: + match = pattern.fullmatch(path) + if match and method == self.command: + params = parse_qs(query, keep_blank_values=True) + status, payload = getattr(self, handler)(body, params, **match.groupdict()) + return status, payload, () + raise ApiError(404, 'not_found', 'Маршрут не найден') + except ApiError as error: + return self._error(error) + except Unavailable as error: + log.warning('event=dependency_unavailable request_id=%s dependency=%s error=%s', + self.request_id, error.dependency, type(error.__cause__).__name__) + return self._error(ApiError(503, 'unavailable', 'Сервис временно недоступен: ' + error.dependency)) + except Exception: + log.exception('event=unhandled_error request_id=%s', self.request_id) + return self._error(ApiError(500, 'internal_error', 'Внутренняя ошибка сервера')) + + def _read_body(self): + """Читает тело целиком до маршрутизации, чтобы keep-alive поток не сбился.""" + header = self.headers.get('Content-Length') + if header is None: + if self.headers.get('Transfer-Encoding'): + self.close_connection = True + raise bad_request('Нужен заголовок Content-Length') + return b'' + if not re.fullmatch(r'[0-9]{1,18}', header): + self.close_connection = True + raise bad_request('Некорректный Content-Length') + length = int(header) + if length > MAX_BODY_BYTES: + self.close_connection = True + if length <= MAX_DISCARD_BYTES: + self._discard(length) + raise ApiError(413, 'payload_too_large', 'Тело запроса больше %d байт' % MAX_BODY_BYTES) + try: + body = self.rfile.read(length) + except OSError: # таймаут или обрыв соединения + body = b'' + if len(body) != length: + self.close_connection = True + raise bad_request('Тело запроса получено не полностью') + return body + + def _discard(self, remaining): + """Дочитывает слишком большое тело по частям, не храня его в памяти. + + Если закрыть соединение с непрочитанными данными, клиент получит обрыв + вместо ответа 413. + """ + try: + while remaining > 0: + chunk = self.rfile.read(min(remaining, 64 * 1024)) + if not chunk: + break + remaining -= len(chunk) + except OSError: + pass + + def _error(self, error): + payload = {'error': {'code': error.code, 'message': error.message, 'request_id': self.request_id}} + headers = (('WWW-Authenticate', error.challenge),) if error.challenge else () + return error.status, payload, headers + + def _send(self, status, payload, headers=()): + """Ответ с JSON-телом; payload=None — ответ без тела (204).""" + if isinstance(payload, Download): + with payload.stream as stream: + self.send_response(status) + self.send_header('Content-Type', payload.row['content_type']) + self.send_header('Content-Length', str(payload.row['size_bytes'])) + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cache-Control', 'private, no-store') + self.end_headers() + if self.command != 'HEAD': + while chunk := stream.read(64 * 1024): + self.wfile.write(chunk) + return + body = b'' if payload is None else json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8') + self.send_response(status) + if payload is not None: + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.send_header('Content-Length', str(len(body))) + self.send_header('X-Request-Id', self.request_id) + # Какая реплика ответила: имя контейнера, стабильно для жизни процесса. + self.send_header('X-Instance-Id', self.server.instance_id) + for name, value in headers: + self.send_header(name, value) + if self.close_connection: + self.send_header('Connection', 'close') + self.end_headers() + if body and self.command != 'HEAD': + self.wfile.write(body) + + def send_error(self, code, message=None, explain=None): + """Ошибки разбора HTTP (до маршрутизации) — тоже в общем формате ошибок.""" + self.request_id = uuid.uuid4().hex + self.close_connection = True + if code == 501: # метод, которого нет в API + error = ApiError(404, 'not_found', 'Маршрут не найден') + else: + error = bad_request('Некорректный HTTP-запрос') + self._send(*self._error(error)) + log.info('request_id=%s event=http_parse_error http_status=%d status=%d', self.request_id, code, error.status) + + def version_string(self): + return 'messenger' # заголовок Server без версии Python + + def log_request(self, code='-', size='-'): + pass # запрос логируется один раз в _handle + + def log_message(self, format, *args): + log.debug('event=http_server detail=%s', format % args) + + # --- общие проверки ----------------------------------------------------- + + def _authenticate(self): + """user_id владельца Bearer-токена. X-User-Id и Basic здесь не принимаются. + + Если Valkey недоступен, sessions.user_id бросает Unavailable и клиент получает 503: + запрос не продолжается как анонимный. + """ + token = validation.parse_bearer(self.headers.get('Authorization')) + user_id = self.server.sessions.user_id(token) if token else None + if user_id is None: + raise ApiError(401, 'unauthorized', 'Нужен действующий токен: Authorization: Bearer ', + challenge=BEARER_CHALLENGE) + self.token = token + self.user_id = user_id + return user_id + + def _accessible_chat(self, user_id, raw_chat_id): + chat_id = validation.parse_uuid(raw_chat_id) + if chat_id is None: + raise bad_request('chat_id должен быть UUID') + chat = self.server.repo.get_chat(chat_id, user_id) + if chat is None: + raise not_found() + return chat + + def _json_body(self, body): + if self.headers.get_content_type() != 'application/json': + raise bad_request('Content-Type должен быть application/json') + return body + + # --- обработчики маршрутов ---------------------------------------------- + + def _live(self, body, query): + return 200, {'status': 'ok'} + + def _ready(self, body, query): + # После SIGTERM реплика сначала перестаёт быть готовой, чтобы балансировщик её исключил. + if not self.server.ready: + raise ApiError(503, 'unavailable', 'Реплика завершает работу') + # Обязательный путь API требует обоих хранилищ: без БД нет данных, без Valkey — сессий. + self.server.repo.ping() + self.server.sessions.ping() + self.server.objects.ping() + return 200, {'status': 'ok'} + + def _create_user(self, body, query): + username, display_name, password = validation.parse_create_user(self._json_body(body)) + try: + return 201, self.server.repo.create_user(username, display_name, hash_password(password)) + except UsernameTaken: + raise ApiError(409, 'conflict', 'Пользователь с таким username уже существует') + + def _create_session(self, body, query): + """Вход: только Authorization: Basic. Ответ 401 одинаковый для любой ошибки входа.""" + if body: + raise bad_request('Тело не нужно: логин и пароль передаются в Authorization: Basic') + credentials = validation.parse_basic(self.headers.get('Authorization')) + user = None + if credentials: + username, password = credentials + user, password_hash = self.server.repo.get_credentials(username) + # Для несуществующего username проверка идёт против фиктивного хеша за то же время. + if not verify_password(password_hash, password): + user = None + if user is None: + raise ApiError(401, 'unauthorized', 'Неверный логин или пароль', challenge=BASIC_CHALLENGE) + token, expires_at = self.server.sessions.create(user['id']) + self.user_id = user['id'] + return 201, {'token': token, 'token_type': 'Bearer', 'expires_at': format_time(expires_at), 'user': user} + + def _delete_session(self, body, query): + self._authenticate() + if not self.server.sessions.revoke(self.token): + # Сессия истекла между проверкой и удалением. + raise ApiError(401, 'unauthorized', 'Сессия уже недействительна', challenge=BEARER_CHALLENGE) + return 204, None + + def _current_user(self, body, query): + user = self.server.repo.get_user(self._authenticate()) + if user is None: + raise ApiError(401, 'unauthorized', 'Пользователь сессии не найден', challenge=BEARER_CHALLENGE) + return 200, user + + def _create_chat(self, body, query): + user_id = self._authenticate() + title, member_ids = validation.parse_create_chat(self._json_body(body)) + if user_id not in member_ids: + raise bad_request('member_ids должен включать вызывающего пользователя') + try: + return 201, self.server.repo.create_chat(title, member_ids) + except UnknownMember: + raise bad_request('member_ids содержит неизвестного пользователя') + + def _list_chats(self, body, query): + user_id = self._authenticate() + scope = 'chats:' + user_id + limit = validation.parse_limit(query) + after = validation.parse_cursor(query, scope) + items, next_key = self.server.repo.list_chats(user_id, after, limit) + return 200, {'items': items, 'next_cursor': validation.encode_cursor(scope, next_key)} + + def _get_chat(self, body, query, chat_id): + return 200, self._accessible_chat(self._authenticate(), chat_id) + + def _create_message(self, body, query, chat_id): + # Повтор с тем же ключом снова проходит аутентификацию и проверку доступа. + user_id = self._authenticate() + chat = self._accessible_chat(user_id, chat_id) + key = validation.parse_idempotency_key(self.headers.get('Idempotency-Key')) + text, ids = validation.parse_create_message(self._json_body(body)) + try: + message, replayed = self.server.repo.add_message( + chat['id'], user_id, text, key, validation.request_fingerprint(text, ids), self.server.idempotency_ttl, ids) + except IdempotencyConflict: + raise ApiError(409, 'conflict', 'Этот Idempotency-Key уже использован с другим телом запроса') + if replayed: + log.info('event=idempotent_replay request_id=%s message_id=%s', self.request_id, message['id']) + return 201, message + + def _list_messages(self, body, query, chat_id): + chat = self._accessible_chat(self._authenticate(), chat_id) + scope = 'messages:' + chat['id'] + limit = validation.parse_limit(query) + after = validation.parse_cursor(query, scope) + items, next_key = self.server.repo.list_messages(chat['id'], after, limit) + return 200, {'items': items, 'next_cursor': validation.encode_cursor(scope, next_key)} + + def _upload(self, body, query, chat_id): + user_id = self._authenticate() + chat = self._accessible_chat(user_id, chat_id) + filename = validation._single(query, 'filename') + if (not filename or not filename.strip() or len(filename) > 255 + or any(c in '/\\' or ord(c) < 32 or 127 <= ord(c) <= 159 for c in filename)): + raise bad_request('filename: безопасное имя из 1–255 символов без путей и управляющих символов') + try: + filename.encode('utf-8') + except UnicodeEncodeError: + raise bad_request('Некорректный filename') + mime = self.headers.get_content_type() + if mime not in ('image/png', 'image/jpeg'): + raise ApiError(415, 'unsupported_media_type', 'Допустимы PNG и JPEG') + lengths = self.headers.get_all('Content-Length', []) + if self.headers.get('Transfer-Encoding') or len(lengths) != 1 or not re.fullmatch(r'[0-9]{1,18}', lengths[0]): + raise bad_request('Нужен один корректный Content-Length') + length = int(lengths[0]) + if length > self.server.max_upload: + raise ApiError(413, 'payload_too_large', 'Максимум 10 MiB') + if not self.server.upload_slots.acquire(timeout=8): + raise ApiError(503, 'unavailable', 'Загрузки заняты; повторите позже') + try: + # Ограниченный временный диск контейнера; оригиналы здесь не сохраняются. + with tempfile.TemporaryDirectory(prefix='upload-') as directory: + path = os.path.join(directory, 'original') + digest = hashlib.sha256() + deadline = time.monotonic() + 10 + with open(path, 'w+b') as file: + remaining = length + while remaining: + self.connection.settimeout(max(0.1, deadline - time.monotonic())) + chunk = self.rfile.read1(min(remaining, 64 * 1024)) + if not chunk or time.monotonic() > deadline: + raise bad_request('Загрузка не завершена в срок') + file.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + file.flush() + try: + result = subprocess.run([sys.executable, '-m', 'messenger.image_check', path, + mime, str(self.server.max_pixels)], timeout=8, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + raise ApiError(415, 'unsupported_media_type', 'Изображение не удалось проверить в срок') + if result.returncode == 13: + raise ApiError(413, 'payload_too_large', 'Слишком много пикселей') + if result.returncode: + raise ApiError(415, 'unsupported_media_type', 'Повреждённое изображение или неверный MIME') + aid = uuid.uuid4() + values = dict(id=aid, chat_id=uuid.UUID(chat['id']), owner_id=uuid.UUID(user_id), + object_key='originals/' + aid.hex, staging_key='staging/' + aid.hex, + filename=filename, content_type=mime, + size_bytes=length, sha256=digest.hexdigest()) + saved = self.server.repo.save_attachment( + values, lambda: self.server.objects.put(values['staging_key'], file, length, mime)) + return 202, saved + except (TimeoutError, ConnectionError): + raise bad_request('Загрузка прервана или превышено время ожидания') + finally: + self.server.upload_slots.release() + + def _accessible_attachment(self, raw_id): + user_id = self._authenticate() + aid = validation.parse_uuid(raw_id) + if aid is None: + raise bad_request('attachment_id должен быть UUID') + row = self.server.repo.get_attachment(aid, user_id) + if row is None: + raise not_found() + return row + + def _attachment(self, body, query, attachment_id): + return 200, attachment(self._accessible_attachment(attachment_id)) + + def _content(self, body, query, attachment_id): + row = self._accessible_attachment(attachment_id) + if row['status'] != 'ready': + raise ApiError(409, 'conflict', 'Вложение ещё не готово или обработка завершилась ошибкой') + name = validation._single(query, 'variant') + if name is None: + name = 'original' + if name != 'original': + variant = next((item for item in row['variants'] if item['name'] == name), None) + if variant is None: + raise not_found() + row = dict(row, **variant) + return 200, Download(self.server.objects.get(row['object_key']), row) diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 0000000..e0d2018 --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,81 @@ +"""Сессии в Valkey — единственное хранилище сессий. + +Ключ: session:, значение: JSON с user_id и сроком. Срок задаётся абсолютным +временем (PXAT), поэтому перезапуск ничего не продлевает. Сохранность при рестарте +обеспечивает AOF с fsync на каждую запись (infra/valkey/valkey.conf). +""" +import hashlib +import json +import secrets +import time +from contextlib import contextmanager + +import valkey +from valkey.backoff import NoBackoff +from valkey.retry import Retry + +from .errors import Unavailable +from .timeutil import from_micros + + +@contextmanager +def _valkey_errors(): + # Любая ошибка Valkey (нет соединения, таймаут, загрузка AOF после рестарта, нехватка памяти) + # превращается в 503. Запрос не продолжается как анонимный и не считается доверенным. + try: + yield + except valkey.exceptions.ValkeyError as error: + raise Unavailable('valkey') from error + + +class SessionStore: + def __init__(self, url, ttl_seconds): + self.ttl_seconds = ttl_seconds + self._pool = valkey.BlockingConnectionPool.from_url( + url, + max_connections=20, # не больше 20 соединений на процесс + timeout=1, # ждать свободное соединение не дольше 1 с + socket_connect_timeout=1, # подключение и ответ — не дольше 1 с + socket_timeout=1, + retry=Retry(NoBackoff(), 1), # один повтор: переподключиться после рестарта Valkey + decode_responses=True, + ) + self._client = valkey.Valkey(connection_pool=self._pool) + + @staticmethod + def _key(token): + # В Valkey хранится только хеш токена: копия базы не даёт готовых токенов. + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def create(self, user_id): + """Новая сессия: (токен, срок действия как datetime).""" + token = secrets.token_urlsafe(32) # 32 случайных байта = 256 бит + expires_ms = int(time.time() * 1000) + self.ttl_seconds * 1000 + value = json.dumps({'user_id': user_id, 'expires_at_ms': expires_ms}) + with _valkey_errors(): + self._client.set(self._key(token), value, pxat=expires_ms) + return token, from_micros(expires_ms * 1000) + + def user_id(self, token): + """user_id владельца действующей сессии или None (неизвестна, отозвана, истекла).""" + with _valkey_errors(): + raw = self._client.get(self._key(token)) + if raw is None: + return None + session = json.loads(raw) + # Valkey удаляет ключ сам; проверка здесь — страховка на случай расхождения часов. + if session['expires_at_ms'] <= time.time() * 1000: + return None + return session['user_id'] + + def revoke(self, token): + """Удаляет сессию; True, если она была.""" + with _valkey_errors(): + return self._client.delete(self._key(token)) == 1 + + def ping(self): + with _valkey_errors(): + self._client.ping() + + def close(self): + self._pool.disconnect() diff --git a/src/messenger/timeutil.py b/src/messenger/timeutil.py new file mode 100644 index 0000000..d38452c --- /dev/null +++ b/src/messenger/timeutil.py @@ -0,0 +1,18 @@ +"""Время в формате контракта: RFC 3339, UTC, микросекунды, суффикс Z.""" +from datetime import datetime, timedelta, timezone + +EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +MICROSECOND = timedelta(microseconds=1) + + +def format_time(value): + return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def to_micros(value): + """Целое число микросекунд с 1970 года: так время хранится в cursor без потери точности.""" + return (value - EPOCH) // MICROSECOND + + +def from_micros(micros): + return EPOCH + timedelta(microseconds=micros) diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 0000000..74da230 --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,219 @@ +"""Разбор и проверка входных данных по contracts/openapi.json и contracts/README.md.""" +import base64 +import binascii +import hashlib +import json +import re + +from .errors import bad_request + +USERNAME = re.compile(r'[a-z0-9_]{3,32}') +UUID = re.compile(r'[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}') +LIMIT = re.compile(r'[0-9]{1,3}') +# Idempotency-Key из контракта: ASCII-символы [A-Za-z0-9._:-], от 1 до 128. +IDEMPOTENCY_KEY = re.compile(r'[A-Za-z0-9._:-]{1,128}') +# token68 из RFC 7235 с разумным ограничением длины. +BEARER_TOKEN = re.compile(r'[A-Za-z0-9\-._~+/]{1,512}=*') + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 +MAX_NAME = 100 +MAX_MEMBERS = 100 +MAX_TEXT = 4000 +MIN_PASSWORD = 12 +MAX_PASSWORD = 128 + + +def parse_uuid(value): + """UUID в каноническом виде (нижний регистр) или None.""" + if isinstance(value, str) and UUID.fullmatch(value): + return value.lower() + return None + + +def _load_object(raw, fields): + """JSON-объект ровно с полями fields: лишние и отсутствующие поля — ошибка.""" + try: + data = json.loads(raw.decode('utf-8')) + except (ValueError, RecursionError): + raise bad_request('Тело запроса должно быть корректным JSON в UTF-8') + if not isinstance(data, dict): + raise bad_request('Тело запроса должно быть JSON-объектом') + for field in fields: + if field not in data: + raise bad_request('Отсутствует обязательное поле ' + field) + if len(data) != len(fields): + raise bad_request('В запросе есть лишние поля') + return data + + +def _string(data, field, max_length, min_length=1): + """Строка из min_length..max_length code points, не только из пробельных символов.""" + value = data[field] + if not isinstance(value, str): + raise bad_request('Поле %s должно быть строкой' % field) + if not value.strip(): + raise bad_request('Поле %s не должно быть пустым' % field) + if not min_length <= len(value) <= max_length: + raise bad_request('Поле %s: от %d до %d символов' % (field, min_length, max_length)) + # NUL нельзя сохранить в тексте PostgreSQL; одиночный суррогат (\ud800) нельзя вернуть в UTF-8. + if '\x00' in value: + raise bad_request('Поле %s содержит недопустимые символы' % field) + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise bad_request('Поле %s содержит недопустимые символы' % field) + return value + + +def parse_create_user(raw): + data = _load_object(raw, ('username', 'display_name', 'password')) + username = data['username'] + if not isinstance(username, str) or not USERNAME.fullmatch(username): + raise bad_request('username: 3-32 символа из набора a-z, 0-9 и _') + display_name = _string(data, 'display_name', MAX_NAME) + password = _string(data, 'password', MAX_PASSWORD, MIN_PASSWORD) + return username, display_name, password + + +def parse_create_chat(raw): + data = _load_object(raw, ('title', 'member_ids')) + title = _string(data, 'title', MAX_NAME) + raw_ids = data['member_ids'] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= MAX_MEMBERS: + raise bad_request('member_ids: список из 1-%d UUID' % MAX_MEMBERS) + member_ids = [parse_uuid(item) for item in raw_ids] + if None in member_ids: + raise bad_request('member_ids должен содержать только UUID') + if len(set(member_ids)) != len(member_ids): + raise bad_request('member_ids не должен содержать повторов') + return title, member_ids + + +def parse_create_message(raw): + try: + data = json.loads(raw.decode('utf-8')) + except (ValueError, RecursionError): + raise bad_request('Тело запроса должно быть корректным JSON в UTF-8') + if not isinstance(data, dict) or 'text' not in data or set(data) - {'text', 'attachment_ids'}: + raise bad_request('Нужен text и необязательный attachment_ids') + text = data['text'] + if not isinstance(text, str) or len(text) > MAX_TEXT or '\x00' in text: + raise bad_request('text: строка до 4000 символов') + try: + text.encode('utf-8') + except UnicodeEncodeError: + raise bad_request('Недопустимые символы text') + ids = data.get('attachment_ids', []) + if not isinstance(ids, list) or len(ids) > 10: + raise bad_request('attachment_ids: список до 10 UUID') + ids = [parse_uuid(value) for value in ids] + if None in ids or len(set(ids)) != len(ids): + raise bad_request('attachment_ids: уникальные UUID') + if not text.strip() and not ids: + raise bad_request('Нужен непустой текст или вложение') + return text, ids + + +def parse_idempotency_key(header): + """Значение заголовка Idempotency-Key; без него или в неверном формате — 400.""" + if header is None: + raise bad_request('Нужен заголовок Idempotency-Key') + if not IDEMPOTENCY_KEY.fullmatch(header): + raise bad_request('Idempotency-Key: от 1 до 128 символов из набора A-Z, a-z, 0-9 и ._:-') + return header + + +def request_fingerprint(text, attachment_ids=()): + """sha256 разобранного тела: порядок ключей и пробелы в JSON не влияют на сравнение повторов.""" + canonical = json.dumps({'text': text, 'attachment_ids': list(attachment_ids)}, + ensure_ascii=False, sort_keys=True, separators=(',', ':')) + return hashlib.sha256(canonical.encode('utf-8')).hexdigest() + + +def _credentials(header, scheme): + """Значение после схемы авторизации (схема без учёта регистра) или None.""" + if not header: + return None + name, _, value = header.strip().partition(' ') + if name.lower() != scheme: + return None + return value.strip() + + +def parse_basic(header): + """(username, password) из Authorization: Basic base64(username:password) или None. + + Пароль в UTF-8 и может содержать двоеточие: делим по первому. Всё, что не может быть + правильной парой логин/пароль, возвращает None — дальше это обычный ответ 401. + """ + value = _credentials(header, 'basic') + if not value: + return None + try: + decoded = base64.b64decode(value, validate=True).decode('utf-8') + except (binascii.Error, ValueError): + return None + username, separator, password = decoded.partition(':') + if not separator or not USERNAME.fullmatch(username): + return None + if not MIN_PASSWORD <= len(password) <= MAX_PASSWORD: + return None + return username, password + + +def parse_bearer(header): + """Токен из Authorization: Bearer или None.""" + value = _credentials(header, 'bearer') + if not value or not BEARER_TOKEN.fullmatch(value): + return None + return value + + +def _single(query, name): + """Значение параметра запроса; None, если параметра нет.""" + values = query.get(name) + if values is None: + return None + if len(values) != 1: + raise bad_request('Параметр %s указан несколько раз' % name) + return values[0] + + +def parse_limit(query): + value = _single(query, 'limit') + if value is None: + return DEFAULT_LIMIT + if not LIMIT.fullmatch(value) or not 1 <= int(value) <= MAX_LIMIT: + raise bad_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT) + return int(value) + + +def encode_cursor(scope, key): + """Непрозрачный для клиента cursor: область действия и ключ (created_at, id) последней записи.""" + if key is None: + return None + raw = '%s|%d|%s' % (scope, key[0], key[1]) + return base64.urlsafe_b64encode(raw.encode('ascii')).decode('ascii').rstrip('=') + + +def parse_cursor(query, scope): + """Ключ (created_at в мкс, id), после которого продолжать; None без cursor. + + scope привязывает cursor к ресурсу: cursor от другого чата — 400. + """ + value = _single(query, 'cursor') + if value is None: + return None + try: + padded = value + '=' * (-len(value) % 4) + raw = base64.b64decode(padded, altchars=b'-_', validate=True).decode('ascii') + cursor_scope, micros, entry_id = raw.split('|') + if cursor_scope != scope or not micros.isdigit() or parse_uuid(entry_id) != entry_id: + raise ValueError + # Ограничение длины защищает от чисел, которые не влезут в дату PostgreSQL. + if len(micros) > 17: + raise ValueError + return int(micros), entry_id + except ValueError: + raise bad_request('Некорректный cursor') diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 0000000..a0c2fe5 --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,307 @@ +"""Bounded at-least-once attachment worker with PostgreSQL lease fencing.""" +import hashlib +import json +import logging +import os +from pathlib import Path +import random +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import uuid + +from .broker import Broker, InvalidDelivery +from .config import read_secret +from .db import create_pool +from .errors import Unavailable +from .image_process import MAX_BYTES, MAX_OUTPUT_BYTES +from .jobs import Jobs +from .objects import ObjectStore + +log = logging.getLogger('messenger.worker') +VARIANTS = {'thumbnail': ('thumbnail.jpg', 'image/jpeg'), 'crop': ('crop.jpg', 'image/jpeg'), + 'optimized': ('optimized.webp', 'image/webp')} + + +class ProcessingError(Exception): + def __init__(self, code, permanent=False): + self.code, self.permanent = code, permanent + super().__init__(code) + + +def env_int(name, default, low, high): + try: + value = int(os.environ.get(name, str(default))) + except ValueError: + raise SystemExit('%s must be an integer' % name) from None + if not low <= value <= high: + raise SystemExit('%s must be in %d..%d' % (name, low, high)) + return value + + +def retry_delay(attempt, rng=None, base_seconds=1, max_seconds=30): + """Capped exponential backoff with equal jitter: [cap/2, cap].""" + cap = min(float(max_seconds), base_seconds * 2.0 ** min(max(attempt - 1, 0), 20)) + return (rng or random).uniform(cap / 2, cap) + + +def cleanup_ready(jobs, store): + count = 0 + for row in jobs.pending_cleanup(limit=100): + # A ready DB record is the authorization to remove its durable staging. + # Repeating S3 DELETE after a lost response is harmless. + store.call('delete_object', Key=row['staging_key']) + jobs.mark_staging_deleted(row['attachment_id'], row['staging_key']) + count += 1 + return count + + +class Worker: + def __init__(self, jobs, store, stop, *, max_attempts=5, lease_seconds=45, + timeout=10, memory_mib=768, temp_root=None, retry_base=1, retry_max=30): + self.jobs, self.store, self.stop = jobs, store, stop + self.max_attempts, self.lease_seconds = max_attempts, lease_seconds + self.timeout, self.memory_mib = timeout, memory_mib + self.temp_root = temp_root + self.retry_base, self.retry_max = retry_base, retry_max + + def fault(self, claim, action): + if not self.jobs.consume_fault(claim['job_id'], action): + return False + log.warning('event=fault_injected job_id=%s action=%s attempt=%s', + claim['job_id'], action, claim['attempt']) + if action.startswith('crash_'): + # Audit committed before termination; restart will not crash twice. + for handler in logging.getLogger().handlers: + handler.flush() + os._exit(86) + return True + + def _download(self, claim, path, deadline): + if not 0 < claim['size_bytes'] <= MAX_BYTES: + raise ProcessingError('image_too_large', permanent=True) + digest, size = hashlib.sha256(), 0 + body = self.store.get(claim['staging_key']) + try: + with open(path, 'wb') as output: + while True: + if self.stop.is_set(): + raise ProcessingError('worker_stopping') + if time.monotonic() >= deadline: + raise ProcessingError('storage_timeout') + chunk = body.read(256 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_BYTES or size > claim['size_bytes']: + raise ProcessingError('staging_integrity', permanent=True) + digest.update(chunk) + output.write(chunk) + finally: + body.close() + if size != claim['size_bytes'] or digest.hexdigest() != claim['sha256']: + raise ProcessingError('staging_integrity', permanent=True) + + def _decode(self, claim, directory): + process = subprocess.Popen( + [sys.executable, '-m', 'messenger.image_process', str(directory / 'source'), str(directory), + claim['content_type'], str(self.memory_mib), str(self.timeout)], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + deadline = time.monotonic() + self.timeout + try: + while process.poll() is None: + if self.stop.wait(0.05): + raise ProcessingError('worker_stopping') + if time.monotonic() >= deadline: + raise ProcessingError('processing_timeout', permanent=True) + if process.returncode: + code = 'image_too_large' if process.returncode == 13 else 'invalid_image' + raise ProcessingError(code, permanent=True) + finally: + if process.poll() is None: + process.kill() + process.wait() + result_path = directory / 'result.json' + if result_path.stat().st_size > 4096: + raise ProcessingError('invalid_result', permanent=True) + with result_path.open(encoding='utf-8') as result_file: + metadata = json.load(result_file) + if not isinstance(metadata, list) or len(metadata) != len(VARIANTS): + raise ProcessingError('invalid_result', permanent=True) + found, total = set(), 0 + for variant in metadata: + name = variant.get('name') + if name not in VARIANTS or name in found: + raise ProcessingError('invalid_result', permanent=True) + filename, mime = VARIANTS[name] + path = directory / filename + total += path.stat().st_size + with path.open('rb') as output: + digest = hashlib.file_digest(output, 'sha256').hexdigest() + if (total > MAX_OUTPUT_BYTES or variant.get('content_type') != mime or + variant.get('size_bytes') != path.stat().st_size or variant.get('sha256') != digest or + any(type(variant.get(dim)) is not int or not 1 <= variant[dim] <= 1280 + for dim in ('width', 'height'))): + raise ProcessingError('invalid_result', permanent=True) + found.add(name) + return metadata + + def process(self, claim): + if self.fault(claim, 'transient'): + raise ProcessingError('injected_transient') + if self.fault(claim, 'permanent'): + raise ProcessingError('invalid_image', permanent=True) + if self.fault(claim, 'stall_processing'): + # Deliberately allow the lease to expire for an audited operator test. + # The stale invocation never writes objects or commits afterwards. + self.stop.wait(self.lease_seconds + 5) + return False + deadline = time.monotonic() + self.lease_seconds - 5 + with tempfile.TemporaryDirectory(prefix='attachment-job-', dir=self.temp_root) as name: + directory = Path(name) + source = directory / 'source' + self._download(claim, source, min(deadline, time.monotonic() + 12)) + variants = self._decode(claim, directory) + attachment_hex = uuid.UUID(str(claim['attachment_id'])).hex + original_key = 'originals/' + attachment_hex + with source.open('rb') as original: + self.store.put(original_key, original, claim['size_bytes'], claim['content_type']) + for variant in variants: + if self.stop.is_set(): + raise ProcessingError('worker_stopping') + if time.monotonic() >= deadline: + raise ProcessingError('lease_deadline') + filename, _ = VARIANTS[variant['name']] + variant['object_key'] = 'variants/%s/v%d/%s' % ( + attachment_hex, claim['pipeline_version'], filename) + with (directory / filename).open('rb') as file: + self.store.put(variant['object_key'], file, variant['size_bytes'], variant['content_type']) + self.fault(claim, 'crash_after_objects') + if not self.jobs.finish(claim, original_key, variants): + log.info('event=lease_lost job_id=%s', claim['job_id']) + return False + self.fault(claim, 'crash_after_ready') + log.info('event=job_ready job_id=%s attachment_id=%s attempt=%s', + claim['job_id'], claim['attachment_id'], claim['attempt']) + # Cleanup does not affect the already committed ready result. Maintenance + # retries it after failures, process death or a lost delete response. + try: + if claim['staging_key'] != original_key: + self.store.call('delete_object', Key=claim['staging_key']) + self.jobs.mark_staging_deleted(claim['attachment_id'], claim['staging_key']) + except Exception as error: + log.warning('event=staging_cleanup_deferred job_id=%s error_type=%s', + claim['job_id'], type(error).__name__) + return True + + def handle(self, broker, message_id, payload, consumer): + claim = self.jobs.claim(payload['job_id'], consumer, self.lease_seconds, self.max_attempts) + if claim is None: + state = self.jobs.delivery_state(payload['job_id']) + if state and (state['status'] == 'failed' or + (state['status'] == 'ready' and state['variant_count'] == 3)): + broker.ack(message_id) + log.info('event=terminal_redelivery job_id=%s status=%s', payload['job_id'], state['status']) + return + log.info('event=job_claimed job_id=%s attempt=%s', claim['job_id'], claim['attempt']) + try: + if (str(claim['attachment_id']) != payload['attachment_id'] or + claim['pipeline_version'] != payload['pipeline_version']): + # Storage metadata always comes from PostgreSQL, never the message. + raise ProcessingError('job_metadata_mismatch', permanent=True) + durable = self.process(claim) + except Exception as error: + if isinstance(error, ProcessingError): + code, permanent = error.code, error.permanent + elif isinstance(error, Unavailable): + code, permanent = 'dependency_unavailable', False + else: + code, permanent = 'processing_failed', False + delay = retry_delay(claim.get('run_attempts', claim['attempt']), + base_seconds=self.retry_base, max_seconds=self.retry_max) + # A failed DB write raises: no ack, and lease recovery owns the work. + durable = self.jobs.fail(claim, code, permanent, self.max_attempts, delay) + log.warning('event=job_error job_id=%s attempt=%s code=%s permanent=%s retry_delay=%.3f error_type=%s', + claim['job_id'], claim['attempt'], code, permanent, delay, type(error).__name__) + if durable: + broker.ack(message_id) + + def consume(self, index): + consumer = '%s-%s-%d' % (socket.gethostname(), uuid.uuid4().hex[:8], index) + broker = Broker() + try: + while not self.stop.is_set(): + try: + broker.ensure_group() + delivery = broker.read(consumer, self.lease_seconds * 1000) + if delivery and not self.stop.is_set(): + self.handle(broker, *delivery, consumer) + except InvalidDelivery as error: + log.error('event=invalid_delivery message_id=%s', error.message_id) + except Exception as error: + # Exception text can include infrastructure URLs: only class + # names and our machine error codes enter worker logs. + log.warning('event=worker_unavailable error_type=%s', type(error).__name__) + self.stop.wait(1) + finally: + broker.close() + + +def main(): + logging.basicConfig(level=os.environ.get('APP_LOG_LEVEL', 'INFO'), + format='%(asctime)s %(levelname)s %(message)s', stream=sys.stdout) + concurrency = env_int('WORKER_CONCURRENCY', 1, 1, 4) + attempts = env_int('JOB_MAX_ATTEMPTS', 5, 1, 20) + timeout = env_int('JOB_TIMEOUT_SECONDS', 10, 1, 30) + lease = env_int('JOB_LEASE_SECONDS', 45, timeout + 15, 600) + memory = env_int('IMAGE_MEMORY_MIB', 768, 256, 2048) + retry_base = env_int('JOB_RETRY_BASE_SECONDS', 1, 1, 60) + retry_max = env_int('JOB_RETRY_MAX_SECONDS', 30, retry_base, 600) + temp_root = Path(os.environ.get('WORKER_TMP_DIR', os.path.join(tempfile.gettempdir(), 'messenger-worker'))) + temp_root.mkdir(parents=True, exist_ok=True) + # The worker owns this dedicated directory. Remove only its own stale temp + # directories from a previous crash, never user paths or arbitrary /tmp data. + for path in temp_root.glob('attachment-job-*'): + if path.is_dir() and not path.is_symlink() and path.resolve().is_relative_to(temp_root.resolve()): + shutil.rmtree(path) + stop = threading.Event() + for signum in (signal.SIGTERM, signal.SIGINT): + signal.signal(signum, lambda *_: stop.set()) + pool = create_pool(read_secret('DATABASE_URL'), max(2, concurrency + 1)) + pool.open(wait=True) + jobs, store = Jobs(pool), ObjectStore() + worker = Worker(jobs, store, stop, max_attempts=attempts, lease_seconds=lease, + timeout=timeout, memory_mib=memory, temp_root=temp_root, + retry_base=retry_base, retry_max=retry_max) + threads = [threading.Thread(target=worker.consume, args=(index,), name='worker-%d' % index) + for index in range(concurrency)] + for thread in threads: + thread.start() + log.info('event=worker_started concurrency=%d prefetch=1 max_attempts=%d lease_seconds=%d timeout_seconds=%d memory_mib=%d', + concurrency, attempts, lease, timeout, memory) + try: + while not stop.is_set(): + try: + recovered = jobs.recover_expired(attempts) + cleaned = cleanup_ready(jobs, store) + if recovered or cleaned: + log.info('event=maintenance recovered=%d staging_deleted=%d', recovered, cleaned) + except Exception as error: + log.warning('event=maintenance_deferred error_type=%s', type(error).__name__) + stop.wait(5) + finally: + stop.set() + for thread in threads: + thread.join() + pool.close() + log.info('event=worker_stopped') + + +if __name__ == '__main__': + main() diff --git a/student-tests/api.py b/student-tests/api.py new file mode 100644 index 0000000..acc69bf --- /dev/null +++ b/student-tests/api.py @@ -0,0 +1,144 @@ +"""Небольшой HTTP-клиент для собственных тестов. Только стандартная библиотека, Python 3.9+. + +Стенд должен быть уже запущен. Адрес берётся из переменной окружения BASE_URL. +По умолчанию 127.0.0.1, а не localhost: в Windows localhost сначала пробует IPv6 +и каждый запрос к IPv4-порту занимает около двух секунд. +""" +import base64 +import json +import os +import urllib.error +import urllib.request +import uuid +from datetime import datetime +from urllib.parse import quote + +BASE_URL = os.environ.get('BASE_URL', 'http://127.0.0.1:8080').rstrip('/') +TIMEOUT = 10 +USER_FIELDS = ('id', 'username', 'display_name', 'created_at') +# Значение по умолчанию для key: новый случайный Idempotency-Key у каждого POST сообщения. +AUTO = object() + + +class Response: + def __init__(self, status, headers, body): + self.status = status + self.headers = headers + self.body = body + self.json = json.loads(body) if body else None + + +def request(method, path, user=None, body=None, raw=None, headers=None, base_url=None, key=AUTO): + """Один запрос. user — словарь с token (Bearer); body — объект для JSON; raw — готовые байты. + + key — Idempotency-Key для POST сообщения: по умолчанию новый случайный, None — без заголовка. + """ + all_headers = {'Accept': 'application/json'} + if user is not None: + all_headers['Authorization'] = 'Bearer ' + user['token'] + if key is AUTO: + key = uuid.uuid4().hex if method == 'POST' and path.endswith('/messages') else None + if key is not None: + all_headers['Idempotency-Key'] = key + if body is not None: + raw = json.dumps(body, ensure_ascii=False).encode('utf-8') + if raw is not None: + all_headers['Content-Type'] = 'application/json' + all_headers.update(headers or {}) + req = urllib.request.Request((base_url or BASE_URL) + path, data=raw, headers=all_headers, method=method) + try: + response = urllib.request.urlopen(req, timeout=TIMEOUT) + except urllib.error.HTTPError as error: + response = error + with response: + result = Response(response.code, response.headers, response.read()) + # Общие требования контракта ко всем ответам. + assert result.headers.get('X-Request-Id'), 'no X-Request-Id in response' + assert result.headers.get('X-Instance-Id'), 'no X-Instance-Id in response' + if result.status == 204: + assert not result.body, '204 must not have a body' + return result + assert result.headers.get_content_type() == 'application/json', 'response is not JSON' + if result.status >= 400: + error = result.json['error'] + assert set(error) == {'code', 'message', 'request_id'}, 'unexpected error fields' + assert error['message'], 'empty error message' + assert error['request_id'] == result.headers['X-Request-Id'], 'request_id differs from header' + return result + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password) в UTF-8.""" + return 'Basic ' + base64.b64encode((username + ':' + password).encode('utf-8')).decode('ascii') + + +def unique_username(prefix='t'): + return prefix + '_' + uuid.uuid4().hex[:16] + + +def new_password(): + # Кириллица и двоеточие: проверяем UTF-8 и деление Basic по первому двоеточию. + return 'Пароль:' + uuid.uuid4().hex + + +def register(username=None, password=None, display_name='Тестовый пользователь'): + body = {'username': username or unique_username(), 'display_name': display_name, 'password': password or new_password()} + return request('POST', '/api/v1/users', body=body), body + + +def login(username, password, base_url=None): + return request('POST', '/api/v1/auth/sessions', headers={'Authorization': basic(username, password)}, + base_url=base_url) + + +def new_user(display_name='Тестовый пользователь', base_url=None): + """Зарегистрированный пользователь с действующим токеном: поля User + token + password.""" + response, body = register(display_name=display_name) + assert response.status == 201, 'user was not created: %d' % response.status + session = login(body['username'], body['password'], base_url=base_url) + assert session.status == 201, 'login failed: %d' % session.status + return dict(response.json, token=session.json['token'], password=body['password']) + + +def public(user): + """Только поля схемы User: так пользователь выглядит в ответах API.""" + return {field: user[field] for field in USER_FIELDS} + + +def new_chat(owner, members, title='Тестовый чат'): + body = {'title': title, 'member_ids': [member['id'] for member in members]} + response = request('POST', '/api/v1/chats', user=owner, body=body) + assert response.status == 201, 'chat was not created: %d' % response.status + return response.json + + +def send(user, chat, text, key=AUTO): + return request('POST', '/api/v1/chats/%s/messages' % chat['id'], user=user, body={'text': text}, key=key) + + +def instance(response): + """Какая реплика API ответила.""" + return response.headers['X-Instance-Id'] + + +def pages(path, user, limit): + """Обходит все страницы списка и возвращает их как список списков.""" + result = [] + cursor = None + while True: + url = '%s?limit=%d' % (path, limit) + if cursor: + url += '&cursor=' + quote(cursor, safe='') + response = request('GET', url, user=user) + assert response.status == 200, 'page was not read: %d' % response.status + result.append(response.json['items']) + cursor = response.json['next_cursor'] + if cursor is None: + return result + assert len(result) < 1000, 'pagination does not finish' + + +def order_key(item): + """Порядок из контракта: (created_at, id) по возрастанию.""" + created_at = datetime.fromisoformat(item['created_at'][:-1] + '+00:00') + return created_at, uuid.UUID(item['id']).int diff --git a/student-tests/fixtures/README.md b/student-tests/fixtures/README.md new file mode 100644 index 0000000..dd7c760 --- /dev/null +++ b/student-tests/fixtures/README.md @@ -0,0 +1,6 @@ +# JPEG fixture + +`sample.jpg` — JPEG-версия публичного синтетического `tests/fixtures/sample.png`, +созданная стандартным JPEG-кодировщиком Windows System.Drawing. Используется только +для проверки JPEG upload/download и побайтной сохранности оригинала. Тесты не +требуют библиотек изображений: fixture уже сохранён в репозитории. diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..73592891d479acb69bcd4b4e866b4d993672534e GIT binary patch literal 7909 zcmbVR3s_TEwmv*kG*l54uuKbAZAF8^$YY#RFjFm95urR1uofdKQlKLvN)-go6sxu( z2#VN2Dk`a3du)vf8LzA`g3)4 z=NZn^U$D@?z~F5?qeX@b7tLR2un-B+(1JbMjM)sv?1gi5=Pdj`|4_dY`m=}%jlXGW zEFm=YHMI0KsNV=vz|8o%OkMb(Xda=dp{1p%t%Xp-*HQ4E(AHm8vnvzdlQi{3H*V42zS6)YRuzpVM| zhj!~fadg`7=|*QyFAmq+XUpe1clqxQ_%bkP-~O<#4;&1SIQ;FAqsPAc{`kqnQ~adl z(ma){G!MW(Eq!gq;?-}>|JYr}Z|{O7 zR^QB8xGv#*;mz4gtvy~E>8g1EsW?0Ps%QCfLlXg8K-qzB9!_(3y z*o3Uyhs8O1A>X8lb(1G3Id;ZX;O-~j8eh3|$VZDxo4kZNMbVnQrW-*FJYinBgr2-n%rX#z1;ck`jSj{j z1T@1Iq&Zr<*L2fVEVPs(cG)qY8o&sWXe>wwen6uGEFi{DWk59pVcEpYa*WCHVLd{i zV{;>HdQ)}hIA3Zif`{f3v(Ju)Imga%xzv<(_))fxxzT4MflVJCIhy4Y^yp^42M{y1 zAUA52U6vzy^c+1xLjIihvFutiZE;4 z&37!N?dg&V0rw7D+6jvSzn2{$((%$xlHWM=WtSl>=3?AeJ1OGaMv90$&Zn1wXj!TJ zG{R;mxKI;=+8Eg??QTsGc-O#&IB9B6r1u6zl<3BLBMk$-J`Q=D!;b`(XDGtqPl|C{ zmQ*CJ6IuI|nrBkP+SVuWPN`8<<~l{rmkxP5U0F>LweM3zsiX7cGTiN$Ia%8zv2M(X z_5ES*o2)?Q*zih>rw0EhXr1 zb&5QA){2NVok1KiyR=nOm(Me(12G|k8!|Vq?Jzj}D9gt|n7MTkMRc7cTbE?1SJ!iH z^F>{~6Dhr(WtnHOnI*P`@rrOW`7n#4T$mdvzv^x%yI2;6j@2RWWD2NK@kJ3)#=UP~ zY(?@!zciJPx&3#jL@H2tp~ln$+RFcm4cs5R(^NL>Hqq`S+-XEFxGF~La$+3;6PTEH zl0KKgOi#-cbW&p5dKFbYIW50nNuD%S@)6*Qe4Q(XnkzLbAT1S0Q^NwPE|%NPgcYmA z$F)=*G{EwmJV9^?C)r)+(g4{=NX>oE=Ot`$zr1@OJ3X}bX20i3lP1T=*1=*x_6*7!J1F8F-YTXlC?c%CQt> zAmZu;O%p;#FBEoOi(V*{q8OwKrFR>ZF%Umd1;Oc&bP|g)k{+?ksqC|FeIjQ#|GZf~ zc+V@A=CI$*`VLv!Lq2tr)eav&oB1Ga_nC>-9|hb%(T#Ct0^Hyvy`aj@)3s8$rEE4u zq;xl&J!I&;>6jv1k)7phVJht!PpyX|IQ&TDbJfAPTL#JbM#8 zMPxZeXz=MM{R$?+2-Xzwp45?V~`0w>#kortxGFXk%gmP+n|k_ zt>`zA#bs7@9-Piq@=UJiC_9y`lCX}JyGkngn6%)HwBwaMeMyQt*<(zKSX!5}&9{b; z8ZsCileHyVA?aqJ^yDBMlBBe;yGBGW%@hIm*1g6P9XS7Ermp{QOd&VjG09YSLxDxt z+NmewjIYDmDQ7dz+!w#Y_I2cDlfW})mlygg{l#D9F9x)IqZ_99%W*_Phw=k~(hdKW zgj^df+9iU7Z3zA{<=!;4a}tRUz3 z_urz}u*V|aNT}E;V{z)*M#&oT$d2Ka2mB@H;_SUXL2=>yRsmTeN)T{+c>@LQ!9A_> zpB@tZ(wqI03xpcRx?&HC2qTwzTxd*x%&Ta2`~Dvett}BYjVH1o`3LXae^Aa7n?Ccn z7;lP{n75fYBJ~eI`%x6Cu=ez4o=OzvMhQEwSzaFre4eocMGNNRjq)a0oU>95&vh(W z>#@9?W4pv36&D8A5+{UGWB#6>6uk4@HPN*|$npxBCoK4>}w_n0q=H3aaMk-9zK( zMV7CLB|0ds<>KII&3fn%ZUFwf(AG#{lDI)(?a!l%g zid(wg#I@=!R3+zF;S`I~6^fx6VR~6dpg5&VNM*#Q^TcKYoc!tP(qJMn*AZ$<)gx4I zCZZDcY}J&VOc|qbZ4xSVXGmMbv4Ns1W&11PCY>S|$fN_V=WTjVHRET&bgbiAobPVP z+owx^z{#So;!Cs7#+}2Qp%R1i%PB(Iap8-^!4(wJTd_1gS1$tu?!%SYx&(iQ|ZLm21u{zW`5Ob za2^-qC}J~vs8w;}EASXdjeCZ_x~7OySW$#^#6iKog$ceHQ#cjo?FDN)?La;I{&elA zEbA&ztKS3a`@$*0g(8M}`M_*4&208cRC#S#ySRqz8hmC7(K<5in1vdfOUW9!4Ol~t zk6yyJBkJeM%Y#zMShehK&+~wmay5he{UPJf?EF9xqkVq(NXyzE9*|AQ{1wn#7+{o> zB!QvjqVdKuT#1P9|Mp?LOfFlh*ih`2cxe>-hJ+$M&r*8~PzpxPptSGJltL`+pGs-Q zIaXASf1Ri&NAVFw+!uIHI>8mFW4GsT+nh|+itg|_A=FwA>|E6)#{iqG5Vx|3)7bpz zh_PePZl#O9B`aR_41aB+bP#YoFQJm&K}FGOB>_WoAqira?rvy*;V{WjFGmgL0v~hG ze@=xV#0?JB>djt_uNv2FS+%IEwarrOU<}@}b`=g zDWa?)y?!6RZv_U`~PP2_ZseA8&@^*9*0zT3Byb!TZau|f4{^QkDA1-vAAQ5yNkRM`%-~4 z{Usza0K|J)Q2k^pxJJ5*B=GstmMS&c#3^XWTZQ4f5!^aJ8wwo))Krk?*#pJxEKW)6 z#04o+ZqVgpDjkmty!y31wnq7azb8Y{7rm}9yje#PomqD{dFLGbzImFp(4{eDS37wg#2 zZ43Jb49cxPlkA5>qUsf3wgk-PMrqs0u3iV@J-#Pw@d<;PD9=&Az()(v$4j;^hOo6Z zWzh4t@~fvX`@t3%1e8D>qi!%BII?TF<$!D)ZqWjh_G&t+d0c2A@c|g20~j}_Pr$e# z^u-c7L^cW!&@Whz%h5^h5a;X3$&E6t#RYc;!ObO81N$Rgaj08$vovq871%?lkGHk#%#69cnZ) zaHGlM#E3$}W+5TU#uK7tn4WW*=M)+Jyl?EL^qEL3?pBpTaQY8L?JB~;P!y((1e!y_~04Yo8^K>8BvgJH(g$}K3eFw2Ix*3z94 zw&K%~z+B1F8H%EC^m%1E_bGJTya=dNE}ye-((Kts*0LMDgevsDD4qgMu`1LMhC{8B zXm0p4QF%hhdfvaUWvR_CvNh?*qN&NkJmuk-8>{B9yTN_n6XhXTVU^r*=wD2lUaVE0 z#oz6}IKimzL<50b=-=qmEpA4ICkzGEs9~yC&Mk>aN z&=tCV#fq#ogTd!!XZ7mP+_aE(av!^DJS;gB&AknxzS;#gy1^Val|Q*GYq%hbhpDIQ zWtg@?ca+1koalq8B9k8}l6H$E_X-}GSjCGAZfLWt7~m8^VZ)-X48@LY)puoW%PGQc zqOQ9k_xHW8p@9fT1B?Ezo@T`ciFLQ2*|*9Z4Xry0lNI42`EZbbVWJ|OEgudxaxETa zUp*IRSABIajK!I0kp+O@w7|PcF-{S!>qNT_RO{I2F2`V^#|2nbq80BuyD(9Lyk?xGVkMlWA_TJn6*(rTat15; zmVB6N8i@^9Qc_$;uI3fm`wxgH=|LcZU-4f4xaZYNXD3B@3+Vc$+m*)TcL-YDE@MlYHzXrF>fMsjE`ipzeHqQL_B z2#Yfev)YVlXVhy|f(8YCGtRIOA7O+|XYEP2W9km=koSu{=N0ZKqrs5$@FVru6LMM& z4xh?7?61D-dsZ~DaFm<&JBoX_K%uwtu9kd@g+cbjV^y5n-Y87102)vj>elCYO_kNp zyK46>8pqNJacYLEQK5j!`BeczhJJTKCwq(*Pl)TL>_ot($eK*LYFTtd4@Q?2NUfW&>6#<(9m5Ci65Xk;JCTjvMF0MV2h{fjp4)YTQ1#(v|xE< zWumJ~$`n~vhRhVuC}UKnZ*+A5G)a0v%k$4F)zAQ@s#*3B^jl$+GRBpt4?eWw&@c-_ d1F~FQy73!Oli)fHvTDoh%Jfas->|8N{{tA}G}izC literal 0 HcmV?d00001 diff --git a/student-tests/race.py b/student-tests/race.py new file mode 100644 index 0000000..0d79b3e --- /dev/null +++ b/student-tests/race.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Проверка защиты от гонок: N клиентов одновременно отправляют сообщения в один чат. + + python student-tests/race.py --output evidence/race.json + +Каждый клиент — отдельный пользователь и отдельный поток. Все потоки стартуют +одновременно, затем история читается постранично и сравнивается с отправленным. +Код возврата 0 — в истории ровно столько уникальных сообщений, сколько отправлено. +""" +import argparse +import json +import sys +import threading +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import api + + +def run_race(clients=10, per_client=10, page_size=7): + users = [api.new_user('Клиент %d' % n) for n in range(clients)] + chat = api.new_chat(users[0], users, title='Проверка гонок') + start = threading.Barrier(clients) + + def client(index): + start.wait() + return [api.send(users[index], chat, 'race client=%d message=%d' % (index, n)) for n in range(per_client)] + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=clients) as pool: + responses = [response for batch in pool.map(client, range(clients)) for response in batch] + elapsed = time.monotonic() - started + + accepted = [response.json for response in responses if response.status == 201] + pages = api.pages('/api/v1/chats/%s/messages' % chat['id'], users[-1], page_size) + history = [message for page in pages for message in page] + per_sender = Counter(message['sender_id'] for message in history) + expected = clients * per_client + summary = { + 'base_url': api.BASE_URL, + 'clients': clients, + 'messages_per_client': per_client, + 'sent': len(responses), + 'accepted_201': len(accepted), + 'history_total': len(history), + 'history_unique_ids': len({message['id'] for message in history}), + 'history_unique_texts': len({message['text'] for message in history}), + 'history_matches_accepted': sorted(m['id'] for m in history) == sorted(m['id'] for m in accepted), + 'senders_in_history': len(per_sender), + 'min_messages_per_sender': min(per_sender.values(), default=0), + 'max_messages_per_sender': max(per_sender.values(), default=0), + 'ordered_by_created_at_and_id': history == sorted(history, key=api.order_key), + 'pages_read': len(pages), + 'page_size': page_size, + 'send_elapsed_seconds': round(elapsed, 3), + } + summary['ok'] = ( + summary['accepted_201'] == expected + and summary['history_total'] == expected + and summary['history_unique_ids'] == expected + and summary['history_unique_texts'] == expected + and summary['history_matches_accepted'] + and summary['senders_in_history'] == clients + and summary['min_messages_per_sender'] == summary['max_messages_per_sender'] == per_client + and summary['ordered_by_created_at_and_id'] + ) + return summary + + +def main(): + parser = argparse.ArgumentParser(description='Concurrent message race check; creates synthetic users and messages') + parser.add_argument('--clients', type=int, default=10) + parser.add_argument('--per-client', type=int, default=10) + parser.add_argument('--output', default=None, help='Optional JSON file for the summary') + args = parser.parse_args() + summary = run_race(args.clients, args.per_client) + text = json.dumps(summary, indent=2) + if args.output: + with open(args.output, 'w', encoding='utf-8') as file: + file.write(text + '\n') + print(text) + print('PASS' if summary['ok'] else 'FAIL') + return 0 if summary['ok'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/test_access.py b/student-tests/test_access.py new file mode 100644 index 0000000..8fcc549 --- /dev/null +++ b/student-tests/test_access.py @@ -0,0 +1,87 @@ +"""Доступ к чатам: аутентификация (кто ты) отдельно от доступа к конкретному чату (можно ли тебе сюда).""" +import unittest +import uuid + +import api + + +class AuthenticationOrderTest(unittest.TestCase): + def test_authentication_is_checked_before_body_and_resource(self): + # Без токена сервер не сообщает ни об ошибке JSON, ни о наличии чата. + self.assertEqual(api.request('POST', '/api/v1/chats', raw=b'{broken').status, 401) + owner = api.new_user() + chat = api.new_chat(owner, [owner]) + path = '/api/v1/chats/%s/messages' % chat['id'] + self.assertEqual(api.request('GET', path).status, 401) + self.assertEqual(api.request('POST', path, raw=b'{broken').status, 401) + + def test_registration_does_not_need_a_token(self): + response, _ = api.register() + self.assertEqual(response.status, 201) + + +class ForeignChatTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.alice, cls.bob, cls.eve = api.new_user('Алиса'), api.new_user('Боб'), api.new_user('Ева') + cls.chat = api.new_chat(cls.alice, [cls.alice, cls.bob]) + cls.path = '/api/v1/chats/' + cls.chat['id'] + assert api.send(cls.alice, cls.chat, 'секрет участников').status == 201 + + def assert_not_found(self, response): + self.assertEqual(response.status, 404) + self.assertEqual(response.json['error']['code'], 'not_found') + + def test_outsider_cannot_get_chat(self): + self.assert_not_found(api.request('GET', self.path, user=self.eve)) + + def test_outsider_cannot_read_history(self): + self.assert_not_found(api.request('GET', self.path + '/messages', user=self.eve)) + + def test_outsider_cannot_write(self): + self.assert_not_found(api.send(self.eve, self.chat, 'чужое сообщение')) + # Даже с неверным телом посторонний видит 404, а не подсказку о валидации. + self.assert_not_found(api.request('POST', self.path + '/messages', user=self.eve, body={'text': ''})) + history = [m for page in api.pages(self.path + '/messages', self.alice, 100) for m in page] + self.assertNotIn('чужое сообщение', [m['text'] for m in history]) + self.assertNotIn(self.eve['id'], [m['sender_id'] for m in history]) + + def test_foreign_chat_looks_like_missing_chat(self): + foreign = api.request('GET', self.path, user=self.eve) + missing = api.request('GET', '/api/v1/chats/' + str(uuid.uuid4()), user=self.eve) + self.assert_not_found(missing) + self.assertEqual(foreign.json['error']['code'], missing.json['error']['code']) + self.assertEqual(foreign.json['error']['message'], missing.json['error']['message']) + + def test_chat_list_contains_only_own_chats(self): + def chat_ids(user): + return [chat['id'] for page in api.pages('/api/v1/chats', user, 100) for chat in page] + + self.assertIn(self.chat['id'], chat_ids(self.alice)) + self.assertIn(self.chat['id'], chat_ids(self.bob)) + self.assertEqual(chat_ids(self.eve), []) + + def test_every_member_can_read_and_write(self): + self.assertEqual(api.request('GET', self.path, user=self.bob).json, self.chat) + sent = api.send(self.bob, self.chat, 'ответ Боба') + self.assertEqual(sent.status, 201) + self.assertEqual(sent.json['sender_id'], self.bob['id']) + + def test_x_user_id_does_not_open_a_foreign_chat(self): + # Ева подставляет id Алисы: доступ решает только владелец токена. + response = api.request('GET', self.path, user=self.eve, headers={'X-User-Id': self.alice['id']}) + self.assert_not_found(response) + + def test_invalid_chat_id_is_400(self): + for suffix in ('', '/messages'): + with self.subTest(suffix=suffix): + response = api.request('GET', '/api/v1/chats/not-a-uuid' + suffix, user=self.alice) + self.assertEqual(response.status, 400) + + def test_unknown_route_is_404(self): + self.assert_not_found(api.request('GET', '/api/v1/unknown', user=self.alice)) + self.assert_not_found(api.request('DELETE', self.path, user=self.alice)) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..6ef5597 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,188 @@ +"""Lab5: asynchronous attachments, variants, ACL and message idempotency.""" +import hashlib +from pathlib import Path +import struct +import sys +import unittest +from urllib.parse import quote +import uuid +import zlib + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, ROOT +from image_probe import jpeg_size + + +def webp_size(data): + """Read the actual WebP canvas from VP8/VP8L/VP8X, independently of API metadata.""" + if data[:4] != b'RIFF' or data[8:12] != b'WEBP': + raise AssertionError('optimized payload is not WebP') + offset = 12 + while offset + 8 <= len(data): + kind = data[offset:offset + 4] + length = int.from_bytes(data[offset + 4:offset + 8], 'little') + payload = data[offset + 8:offset + 8 + length] + if len(payload) != length: + raise AssertionError('truncated WebP chunk') + if kind == b'VP8X' and length >= 10: + return (1 + int.from_bytes(payload[4:7], 'little'), + 1 + int.from_bytes(payload[7:10], 'little')) + if kind == b'VP8 ' and length >= 10 and payload[3:6] == b'\x9d\x01\x2a': + return (int.from_bytes(payload[6:8], 'little') & 0x3fff, + int.from_bytes(payload[8:10], 'little') & 0x3fff) + if kind == b'VP8L' and length >= 5 and payload[0] == 0x2f: + bits = int.from_bytes(payload[1:5], 'little') + return (1 + (bits & 0x3fff), 1 + ((bits >> 14) & 0x3fff)) + offset += 8 + length + (length & 1) + raise AssertionError('WebP dimensions not found') + + +def small_png(width=13, height=7): + """A transparent RGBA fixture; no host imaging library is required.""" + def chunk(kind, data): + return (struct.pack('>I', len(data)) + kind + data + + struct.pack('>I', zlib.crc32(kind + data))) + scanlines = b''.join(b'\0' + b'\x14\x78\xdc\0' * width for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 6, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(scanlines)) + chunk(b'IEND', b'')) + + +class AttachmentsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.c = Client('http://127.0.0.1:8080' if not __import__('os').environ.get('BASE_URL') + else __import__('os').environ['BASE_URL']) + cls.a, cls.b, cls.eve, cls.chat = cls.c.bootstrap() + cls.png = (ROOT / 'tests/fixtures/sample.png').read_bytes() + cls.path = '/api/v1/chats/' + cls.chat['id'] + '/attachments' + + def upload(self, raw=None, mime='image/png', filename='photo.png', status=202, actor=None, headers=None): + return self.c.call('POST', self.path + '?filename=' + quote(filename, safe=''), status, + actor=actor or self.a, raw=self.png if raw is None else raw, + headers={'Content-Type': mime, **(headers or {})}, + schema='Attachment' if status == 202 else None) + + def ready(self, attachment): + self.assertEqual((attachment['status'], attachment['variants'], attachment['error_code']), + ('queued', [], None)) + return self.c.wait_ready(self.b, attachment) + + def assert_variants(self, attachment, thumbnail=(256, 160), optimized=(320, 200)): + variants = {item['name']: item for item in attachment['variants']} + self.assertEqual(set(variants), {'thumbnail', 'crop', 'optimized'}) + self.assertEqual(len(attachment['variants']), 3, 'duplicate variants') + for name, expected_size, mime, probe in ( + ('thumbnail', thumbnail, 'image/jpeg', jpeg_size), + ('crop', (128, 128), 'image/jpeg', jpeg_size), + ('optimized', optimized, 'image/webp', webp_size)): + with self.subTest(variant=name): + metadata = variants[name] + result = self.c.call('GET', '/api/v1/attachments/' + attachment['id'] + + '/content?variant=' + name, actor=self.b) + self.assertEqual(result['headers'].get_content_type(), mime) + self.assertEqual(metadata['content_type'], mime) + self.assertEqual((metadata['width'], metadata['height']), expected_size) + self.assertEqual(probe(result['bytes']), expected_size) + self.assertEqual(metadata['size_bytes'], len(result['bytes'])) + self.assertEqual(metadata['sha256'], hashlib.sha256(result['bytes']).hexdigest()) + + def test_original_and_metadata_and_acl(self): + queued = self.upload(filename='фото 👋.png')['json'] + a = self.ready(queued) + self.assertEqual(a['sha256'], hashlib.sha256(self.png).hexdigest()) + self.assertEqual(a['size_bytes'], len(self.png)) + self.assertEqual((a['status'], a['error_code']), ('ready', None)) + self.assert_variants(a) + path = '/api/v1/attachments/' + a['id'] + self.assertEqual(self.c.call('GET', path, actor=self.b, schema='Attachment')['json'], a) + result = self.c.call('GET', path + '/content?variant=original', actor=self.b) + self.assertEqual(result['bytes'], self.png) + self.assertEqual(result['headers'].get_content_type(), 'image/png') + for suffix in ('', '/content'): + self.c.call('GET', path + suffix, 404, actor=self.eve) + self.c.call('GET', path + suffix, 401) + self.c.call('GET', path + '/content?variant=unknown', 404, actor=self.a) + + def test_jpeg_original(self): + jpeg = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes() + a = self.ready(self.upload(raw=jpeg, mime='image/jpeg', filename='photo.jpg')['json']) + got = self.c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=self.b) + self.assertEqual(got['bytes'], jpeg) + self.assertEqual(got['headers'].get_content_type(), 'image/jpeg') + self.assert_variants(a) + + def test_small_transparent_image_does_not_upscale_thumbnail_or_optimized(self): + data = small_png() + a = self.ready(self.upload(raw=data, filename='transparent.png')['json']) + self.assert_variants(a, thumbnail=(13, 7), optimized=(13, 7)) + got = self.c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=self.a) + self.assertEqual(got['bytes'], data) + + def test_large_optimized_image_is_bounded(self): + data = small_png(1600, 1000) + a = self.ready(self.upload(raw=data, filename='large.png')['json']) + self.assert_variants(a, thumbnail=(256, 160), optimized=(1280, 800)) + + def test_exif_orientation_applies_to_variants_and_original_stays_identical(self): + jpeg = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes() + # One valid little-endian TIFF IFD with Orientation=6 (90 degrees CW). + exif = (b'Exif\0\0II\x2a\0\x08\0\0\0\x01\0' + + struct.pack('H', len(exif) + 2) + exif + jpeg[2:] + a = self.ready(self.upload(raw=source, mime='image/jpeg', filename='rotated.jpg')['json']) + self.assert_variants(a, thumbnail=(160, 256), optimized=(200, 320)) + original = self.c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=self.a) + self.assertEqual(original['bytes'], source) + + def test_corrupt_and_wrong_mime(self): + for raw, mime in ((b'not an image', 'image/png'), (self.png[:40], 'image/png'), + (self.png, 'image/jpeg'), (self.png, 'text/plain'), (b'', 'image/png')): + with self.subTest(mime=mime, size=len(raw)): + self.upload(raw=raw, mime=mime, status=415) + + def test_filename(self): + for name in ('', ' ', '../x.png', 'a/b.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'a\x7f.png', 'я' * 256): + with self.subTest(name=repr(name)): + self.upload(filename=name, status=400) + + def test_byte_limit_before_body(self): + # Большой Content-Length с одним байтом: сервер отклоняет до чтения/выделения памяти. + self.upload(raw=b'x', status=413, headers={'Content-Length': '10485761'}) + + def test_pixel_limit_before_decode(self): + ihdr = struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0) + chunk = b'IHDR' + ihdr + png = (b'\x89PNG\r\n\x1a\n' + struct.pack('>I', len(ihdr)) + chunk + + struct.pack('>I', zlib.crc32(chunk)) + self.png[33:]) + self.upload(raw=png, status=413) + + def test_foreign_chat_upload(self): + self.upload(actor=self.eve, status=404) + + def test_attachment_only_and_idempotency(self): + aid = self.upload()['json']['id'] + key = uuid.uuid4().hex + first = self.c.message(self.b, self.chat, '', key=key, attachments=[aid])['json'] + self.assertEqual(first['attachment_ids'], [aid]) + self.assertEqual(self.c.message(self.b, self.chat, '', key=key, attachments=[aid])['json'], first) + path = '/api/v1/chats/' + self.chat['id'] + '/messages' + self.c.call('POST', path, 409, actor=self.b, body={'text': 'другое', 'attachment_ids': [aid]}, + headers={'Idempotency-Key': key}) + key = uuid.uuid4().hex + first = self.c.message(self.a, self.chat, 'text', key=key)['json'] + self.assertEqual(self.c.message(self.a, self.chat, 'text', key=key, attachments=[])['json'], first) + + def test_unknown_foreign_and_duplicate_ids(self): + aid = self.upload()['json']['id'] + other = self.c.call('POST', '/api/v1/chats', 201, actor=self.a, + body={'title': 'другой', 'member_ids': [self.a['id']]}, schema='Chat')['json'] + for chat, ids, status in ((other, [aid], 404), (self.chat, [str(uuid.uuid4())], 404), + (self.chat, [aid, aid], 400), (self.chat, [aid] * 11, 400), + (self.chat, ['bad'], 400)): + self.c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', status, actor=self.a, + body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..9a918dc --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,39 @@ +"""Балансировка: запросы одной сессии обслуживают разные реплики, состояние у них общее.""" +import time +import unittest + +import api + + +class BalancingTest(unittest.TestCase): + def test_one_session_is_served_by_several_replicas(self): + user = api.new_user('Балансировка') + responses = [api.request('GET', '/api/v1/users/me', user=user) for _ in range(20)] + self.assertEqual({response.status for response in responses}, {200}) + # Сессия общая (Valkey), поэтому любая реплика узнаёт токен без повторного входа. + self.assertGreaterEqual(len({api.instance(response) for response in responses}), 2) + + def test_message_written_on_one_replica_is_read_on_another(self): + alice, bob = api.new_user('Алиса'), api.new_user('Боб') + chat = api.new_chat(alice, [alice, bob]) + sent = api.send(alice, chat, 'запись на одной реплике') + self.assertEqual(sent.status, 201) + writer = api.instance(sent) + path = '/api/v1/chats/%s/messages' % chat['id'] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + read = api.request('GET', path, user=bob) + if api.instance(read) != writer: + # Подтверждённое сообщение сразу видно через другую реплику. + self.assertEqual(read.json['items'], [sent.json]) + return + self.fail('no response from another replica within 5 s') + + def test_errors_also_carry_instance_id(self): + for method, path in (('GET', '/api/v1/users/me'), ('GET', '/api/v1/unknown'), ('GET', '/health/live')): + with self.subTest(path=path): + self.assertTrue(api.instance(api.request(method, path))) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 0000000..b3a2920 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,62 @@ +"""Параллельные запросы не должны терять, дублировать или портить данные.""" +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor + +import api +from race import run_race + + +class ConcurrencyTest(unittest.TestCase): + def test_100_messages_from_10_clients(self): + summary = run_race(clients=10, per_client=10) + self.assertEqual(summary['accepted_201'], 100) + self.assertEqual(summary['history_total'], 100) + self.assertEqual(summary['history_unique_ids'], 100) + self.assertEqual(summary['history_unique_texts'], 100) + self.assertTrue(summary['history_matches_accepted']) + self.assertEqual(summary['senders_in_history'], 10) + self.assertEqual((summary['min_messages_per_sender'], summary['max_messages_per_sender']), (10, 10)) + self.assertTrue(summary['ordered_by_created_at_and_id']) + + def test_same_username_registered_in_parallel_wins_once(self): + # Уникальность username держит индекс PostgreSQL: из одновременных INSERT проходит один, + # остальные получают нарушение уникальности, и API отвечает 409. + attempts = 20 + username = api.unique_username() + passwords = ['Гонка-пароль-%02d' % n for n in range(attempts)] + start = threading.Barrier(attempts) + + def register(index): + start.wait() + body = {'username': username, 'display_name': 'Гонка %d' % index, 'password': passwords[index]} + return api.request('POST', '/api/v1/users', body=body) + + with ThreadPoolExecutor(max_workers=attempts) as pool: + responses = list(pool.map(register, range(attempts))) + statuses = sorted(response.status for response in responses) + self.assertEqual(statuses, [201] + [409] * (attempts - 1)) + # Победитель один, и войти можно только с его паролем. + winner = next(index for index, response in enumerate(responses) if response.status == 201) + logins = [api.login(username, password).status for password in passwords] + self.assertEqual(logins, [201 if index == winner else 401 for index in range(attempts)]) + + def test_chats_created_in_parallel_are_all_listed(self): + user = api.new_user() + count = 30 + start = threading.Barrier(count) + + def create(index): + start.wait() + return api.new_chat(user, [user], title='Параллельный чат %d' % index) + + with ThreadPoolExecutor(max_workers=count) as pool: + created = list(pool.map(create, range(count))) + listed = [chat for page in api.pages('/api/v1/chats', user, 7) for chat in page] + self.assertEqual(len(listed), count) + self.assertEqual({chat['id'] for chat in listed}, {chat['id'] for chat in created}) + self.assertEqual(listed, sorted(listed, key=api.order_key)) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..81b5b23 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,106 @@ +"""Идемпотентность отправки сообщения: Idempotency-Key общий для всех реплик.""" +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +import api + + +class IdempotencyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.alice, cls.bob = api.new_user('Алиса'), api.new_user('Боб') + cls.chat = api.new_chat(cls.alice, [cls.alice, cls.bob]) + cls.path = '/api/v1/chats/%s/messages' % cls.chat['id'] + + def history(self): + return [m for page in api.pages(self.path, self.alice, 100) for m in page] + + def test_replay_returns_the_same_message(self): + key = uuid.uuid4().hex + first = api.send(self.alice, self.chat, 'повтор ' + key, key=key) + second = api.send(self.alice, self.chat, 'повтор ' + key, key=key) + self.assertEqual((first.status, second.status), (201, 201)) + self.assertEqual(second.json, first.json) + self.assertEqual([m['text'] for m in self.history()].count('повтор ' + key), 1) + + def test_same_key_with_another_body_is_409(self): + key = uuid.uuid4().hex + self.assertEqual(api.send(self.alice, self.chat, 'первое тело', key=key).status, 201) + response = api.send(self.alice, self.chat, 'другое тело', key=key) + self.assertEqual(response.status, 409) + self.assertEqual(response.json['error']['code'], 'conflict') + self.assertNotIn('другое тело', [m['text'] for m in self.history()]) + + def test_json_formatting_does_not_matter(self): + # Пробелы и запись Unicode в JSON не меняют смысл тела: это тот же повтор. + key = uuid.uuid4().hex + first = api.request('POST', self.path, user=self.alice, raw='{"text":"Привет"}'.encode(), key=key) + second = api.request('POST', self.path, user=self.alice, raw=b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }', key=key) + self.assertEqual((first.status, second.status), (201, 201)) + self.assertEqual(second.json, first.json) + + def test_missing_key_is_400(self): + response = api.send(self.alice, self.chat, 'без ключа', key=None) + self.assertEqual(response.status, 400) + self.assertNotIn('без ключа', [m['text'] for m in self.history()]) + + def test_invalid_keys_are_400(self): + # Кириллица уходит в заголовок байтами UTF-8, как её отправил бы обычный клиент. + cyrillic = 'кириллица'.encode('utf-8').decode('latin-1') + for key in ('', 'a' * 129, cyrillic, 'with space', 'semi;colon', 'slash/key'): + with self.subTest(key=key): + self.assertEqual(api.send(self.alice, self.chat, 'ключ', key=key).status, 400) + + def test_key_format_limits_are_accepted(self): + # Свой чат на каждый запуск: область ключа включает чат, поэтому ключи не пересекаются с прошлыми. + chat = api.new_chat(self.alice, [self.alice], title='Форматы ключей') + for key in ('a' * 128, 'A.b_c:d-1', '0'): + with self.subTest(key=key[:20]): + self.assertEqual(api.send(self.alice, chat, 'формат ' + key, key=key).status, 201) + + def test_key_scope_is_user_and_chat(self): + key = uuid.uuid4().hex + other_chat = api.new_chat(self.alice, [self.alice], title='Другой чат') + first = api.send(self.alice, self.chat, 'одинаковый ключ', key=key) + in_other_chat = api.send(self.alice, other_chat, 'одинаковый ключ', key=key) + by_other_user = api.send(self.bob, self.chat, 'одинаковый ключ', key=key) + self.assertEqual({first.status, in_other_chat.status, by_other_user.status}, {201}) + self.assertEqual(len({first.json['id'], in_other_chat.json['id'], by_other_user.json['id']}), 3) + + def test_failed_validation_is_not_remembered(self): + # Ответ 400 не фиксируется: тот же ключ можно использовать с исправленным телом. + key = uuid.uuid4().hex + self.assertEqual(api.send(self.alice, self.chat, ' ', key=key).status, 400) + self.assertEqual(api.send(self.alice, self.chat, 'исправлено', key=key).status, 201) + + def test_replay_checks_the_session_again(self): + user = api.new_user('Выход') + chat = api.new_chat(user, [user]) + key = uuid.uuid4().hex + self.assertEqual(api.send(user, chat, 'до выхода', key=key).status, 201) + api.request('DELETE', '/api/v1/auth/sessions/current', user=user) + self.assertEqual(api.send(user, chat, 'до выхода', key=key).status, 401) + + def test_parallel_requests_with_one_key_create_one_message(self): + # Одновременные повторы расходятся по разным репликам и всё равно создают одно сообщение. + key = uuid.uuid4().hex + text = 'параллельно ' + key + attempts = 20 + start = threading.Barrier(attempts) + + def post(_): + start.wait() + return api.send(self.alice, self.chat, text, key=key) + + with ThreadPoolExecutor(max_workers=attempts) as pool: + responses = list(pool.map(post, range(attempts))) + self.assertEqual({response.status for response in responses}, {201}) + self.assertEqual(len({response.json['id'] for response in responses}), 1) + self.assertGreaterEqual(len({api.instance(response) for response in responses}), 2) + self.assertEqual([m['text'] for m in self.history()].count(text), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py new file mode 100644 index 0000000..d2509d1 --- /dev/null +++ b/student-tests/test_pagination.py @@ -0,0 +1,127 @@ +"""Keyset-пагинация истории сообщений и списка чатов.""" +import unittest +from urllib.parse import quote + +import api + + +class MessagePaginationTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.user = api.new_user() + cls.chat = api.new_chat(cls.user, [cls.user]) + cls.path = '/api/v1/chats/%s/messages' % cls.chat['id'] + cls.sent = [api.send(cls.user, cls.chat, 'сообщение %d' % n).json for n in range(25)] + + def get(self, query): + return api.request('GET', self.path + query, user=self.user) + + def test_walk_over_several_pages(self): + pages = api.pages(self.path, self.user, 10) + self.assertEqual([len(page) for page in pages], [10, 10, 5]) + history = [message for page in pages for message in page] + # Ничего не потеряно, не повторено и не переставлено. + self.assertEqual(history, self.sent) + self.assertEqual(history, sorted(history, key=api.order_key)) + self.assertEqual(len({message['id'] for message in history}), 25) + + def test_one_message_per_page(self): + pages = api.pages(self.path, self.user, 1) + self.assertEqual(len(pages), 25) + self.assertEqual([page[0] for page in pages], self.sent) + + def test_last_full_page_has_no_cursor(self): + # 25 сообщений по 5: на пятой странице следующей записи нет, cursor сразу null. + pages = api.pages(self.path, self.user, 5) + self.assertEqual([len(page) for page in pages], [5, 5, 5, 5, 5]) + + def test_default_and_maximum_limit(self): + self.assertEqual(len(self.get('').json['items']), 25) + everything = self.get('?limit=100').json + self.assertEqual(everything['items'], self.sent) + self.assertIsNone(everything['next_cursor']) + + def test_limit_can_change_between_pages(self): + first = self.get('?limit=3').json + second = self.get('?limit=20&cursor=' + quote(first['next_cursor'], safe='')).json + self.assertEqual(first['items'] + second['items'], self.sent[:23]) + self.assertIsNotNone(second['next_cursor']) + + def test_invalid_limit_is_400(self): + for limit in ('0', '101', '-1', '1.5', 'abc', '', '1e2', '٣'): + with self.subTest(limit=limit): + response = self.get('?limit=' + quote(limit)) + self.assertEqual(response.status, 400) + self.assertEqual(response.json['error']['code'], 'invalid_request') + + def test_invalid_cursor_is_400(self): + for cursor in ('garbage', '', '!!!', 'AAAA', quote('курсор')): + with self.subTest(cursor=cursor): + self.assertEqual(self.get('?cursor=' + cursor).status, 400) + + def test_cursor_from_another_chat_is_400(self): + other = api.new_chat(self.user, [self.user], title='Другой чат') + for n in range(3): + api.send(self.user, other, 'в другом чате %d' % n) + other_cursor = api.request('GET', '/api/v1/chats/%s/messages?limit=1' % other['id'], user=self.user).json['next_cursor'] + self.assertIsNotNone(other_cursor) + self.assertEqual(self.get('?cursor=' + quote(other_cursor, safe='')).status, 400) + # Cursor списка чатов тоже не подходит к истории сообщений. + chats_cursor = api.request('GET', '/api/v1/chats?limit=1', user=self.user).json['next_cursor'] + self.assertEqual(self.get('?cursor=' + quote(chats_cursor, safe='')).status, 400) + + +class DefaultLimitTest(unittest.TestCase): + def test_default_limit_is_50(self): + user = api.new_user() + chat = api.new_chat(user, [user]) + path = '/api/v1/chats/%s/messages' % chat['id'] + for n in range(55): + api.send(user, chat, 'сообщение %d' % n) + first = api.request('GET', path, user=user).json + self.assertEqual(len(first['items']), 50) + rest = api.request('GET', path + '?cursor=' + quote(first['next_cursor'], safe=''), user=user).json + self.assertEqual(len(rest['items']), 5) + self.assertIsNone(rest['next_cursor']) + + +class GrowingHistoryTest(unittest.TestCase): + def test_empty_history(self): + user = api.new_user() + chat = api.new_chat(user, [user]) + page = api.request('GET', '/api/v1/chats/%s/messages' % chat['id'], user=user).json + self.assertEqual(page, {'items': [], 'next_cursor': None}) + + def test_messages_sent_during_walk_are_not_lost(self): + alice, bob = api.new_user(), api.new_user() + chat = api.new_chat(alice, [alice, bob]) + path = '/api/v1/chats/%s/messages' % chat['id'] + before = [api.send(alice, chat, 'до %d' % n).json for n in range(4)] + first = api.request('GET', path + '?limit=2', user=bob).json + # Пока Боб читает, Алиса дописывает; cursor Боба продолжает работать у любого участника. + after = [api.send(alice, chat, 'после %d' % n).json for n in range(3)] + rest = api.request('GET', path + '?limit=100&cursor=' + quote(first['next_cursor'], safe=''), user=alice).json + self.assertEqual(first['items'] + rest['items'], before + after) + + +class ChatListPaginationTest(unittest.TestCase): + def test_chats_are_paged_in_creation_order(self): + user = api.new_user() + created = [api.new_chat(user, [user], title='Чат %d' % n) for n in range(5)] + pages = api.pages('/api/v1/chats', user, 2) + self.assertEqual([len(page) for page in pages], [2, 2, 1]) + chats = [chat for page in pages for chat in page] + self.assertEqual(chats, created) + self.assertEqual(chats, sorted(chats, key=api.order_key)) + + def test_cursor_of_another_user_is_400(self): + alice, bob = api.new_user(), api.new_user() + for n in range(2): + api.new_chat(alice, [alice, bob], title='Общий %d' % n) + alice_cursor = api.request('GET', '/api/v1/chats?limit=1', user=alice).json['next_cursor'] + response = api.request('GET', '/api/v1/chats?limit=1&cursor=' + quote(alice_cursor, safe=''), user=bob) + self.assertEqual(response.status, 400) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 0000000..4bdbfd5 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,160 @@ +"""Вход по Basic, Bearer-сессии, выход. X-User-Id из lab1 больше не даёт доступа.""" +import base64 +import unittest +import uuid +from datetime import datetime, timedelta, timezone + +import api + +ME = '/api/v1/users/me' +SESSIONS = '/api/v1/auth/sessions' + + +def b64(raw): + return base64.b64encode(raw).decode('ascii') + + +class LoginTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + response, cls.registration = api.register(display_name='Вход') + assert response.status == 201 + cls.user = response.json + + def assert_login_rejected(self, response): + self.assertEqual(response.status, 401) + self.assertEqual(response.json['error']['code'], 'unauthorized') + self.assertEqual(response.headers['WWW-Authenticate'], 'Basic realm="messenger", charset="UTF-8"') + return response.json['error']['message'] + + def test_successful_login(self): + response = api.login(self.registration['username'], self.registration['password']) + self.assertEqual(response.status, 201) + session = response.json + self.assertEqual(set(session), {'token', 'token_type', 'expires_at', 'user'}) + self.assertEqual(session['token_type'], 'Bearer') + self.assertEqual(session['user'], self.user) + # Непрозрачный случайный токен: 256 бит в base64url — 43 символа; это не id и не username. + self.assertGreaterEqual(len(session['token']), 43) + self.assertNotIn(self.user['id'], session['token']) + self.assertNotIn(self.user['username'], session['token']) + expires_at = datetime.fromisoformat(session['expires_at'][:-1] + '+00:00') + self.assertGreater(expires_at, datetime.now(timezone.utc) + timedelta(seconds=60)) + me = api.request('GET', ME, headers={'Authorization': 'Bearer ' + session['token']}) + self.assertEqual(me.json, self.user) + + def test_each_login_gives_a_new_token(self): + first = api.login(self.registration['username'], self.registration['password']).json['token'] + second = api.login(self.registration['username'], self.registration['password']).json['token'] + self.assertNotEqual(first, second) + for token in (first, second): + self.assertEqual(api.request('GET', ME, headers={'Authorization': 'Bearer ' + token}).status, 200) + + def test_registration_never_returns_password(self): + response, _ = api.register() + self.assertEqual(set(response.json), {'id', 'username', 'display_name', 'created_at'}) + + def test_wrong_password(self): + self.assert_login_rejected(api.login(self.registration['username'], self.registration['password'] + 'x')) + + def test_unknown_user_gets_the_same_answer(self): + wrong_password = self.assert_login_rejected(api.login(self.registration['username'], 'Wrong-password-123')) + unknown_user = self.assert_login_rejected(api.login(api.unique_username(), 'Wrong-password-123')) + self.assertEqual(wrong_password, unknown_user) + + def test_malformed_basic_header(self): + for value in ( + 'Basic', + 'Basic !!!not-base64!!!', + 'Basic ' + b64(b'no-colon-here'), + 'Basic ' + b64(b'\xff\xfe:invalid-utf8-password'), + 'Basic ' + b64(b'Upper_Case:Valid-password-123'), + 'Basic ' + b64(b'short:pw'), + 'Digest abc', + 'Bearer ' + uuid.uuid4().hex, + ): + with self.subTest(value=value): + response = api.request('POST', SESSIONS, headers={'Authorization': value}) + self.assert_login_rejected(response) + self.assert_login_rejected(api.request('POST', SESSIONS)) + + def test_password_with_colons_and_unicode(self): + password = 'Пароль:с:двоеточиями 👋' + response, body = api.register(password=password) + self.assertEqual(response.status, 201) + self.assertEqual(api.login(body['username'], password).status, 201) + self.assert_login_rejected(api.login(body['username'], 'Пароль:с:двоеточиями')) + + def test_credentials_in_json_body_are_not_accepted(self): + body = {'username': self.registration['username'], 'password': self.registration['password']} + response = api.request('POST', SESSIONS, body=body) + self.assertIn(response.status, (400, 401)) + self.assertNotIn('token', response.json) + + +class BearerTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.alice = api.new_user('Алиса') + cls.bob = api.new_user('Боб') + + def assert_bearer_rejected(self, response): + self.assertEqual(response.status, 401) + self.assertEqual(response.json['error']['code'], 'unauthorized') + self.assertEqual(response.headers['WWW-Authenticate'], 'Bearer realm="messenger"') + + def test_protected_endpoints_need_a_token(self): + for method, path in (('GET', ME), ('GET', '/api/v1/chats'), ('POST', '/api/v1/chats'), + ('GET', '/api/v1/chats/%s/messages' % uuid.uuid4()), ('DELETE', SESSIONS + '/current')): + with self.subTest(method=method, path=path): + self.assert_bearer_rejected(api.request(method, path)) + + def test_unknown_or_malformed_token(self): + for value in ('Bearer ' + uuid.uuid4().hex * 2, 'Bearer', 'Bearer a b', 'Bearer ' + 'x' * 600, + 'Bearer ' + self.alice['id'], 'Token ' + self.alice['token']): + with self.subTest(value=value[:40]): + self.assert_bearer_rejected(api.request('GET', ME, headers={'Authorization': value})) + + def test_basic_is_rejected_on_business_endpoints(self): + value = api.basic(self.alice['username'], self.alice['password']) + self.assert_bearer_rejected(api.request('GET', ME, headers={'Authorization': value})) + + def test_x_user_id_is_ignored(self): + self.assert_bearer_rejected(api.request('GET', ME, headers={'X-User-Id': self.alice['id']})) + # Заголовок X-User-Id не меняет владельца сессии. + me = api.request('GET', ME, user=self.alice, headers={'X-User-Id': self.bob['id']}) + self.assertEqual(me.json, api.public(self.alice)) + + def test_token_selects_the_user(self): + self.assertEqual(api.request('GET', ME, user=self.alice).json, api.public(self.alice)) + self.assertEqual(api.request('GET', ME, user=self.bob).json, api.public(self.bob)) + + def test_scheme_name_is_case_insensitive(self): + # RFC 7235: имя схемы авторизации не зависит от регистра. + response = api.request('GET', ME, headers={'Authorization': 'bearer ' + self.alice['token']}) + self.assertEqual(response.status, 200) + + +class LogoutTest(unittest.TestCase): + def test_logout_revokes_only_the_current_token(self): + user = api.new_user('Выход') + other = api.login(user['username'], user['password']).json['token'] + response = api.request('DELETE', SESSIONS + '/current', user=user) + self.assertEqual(response.status, 204) + self.assertEqual(response.body, b'') + # Отозванный токен больше не работает, повторный выход — тоже 401. + self.assertEqual(api.request('GET', ME, user=user).status, 401) + self.assertEqual(api.request('DELETE', SESSIONS + '/current', user=user).status, 401) + # Другая сессия того же пользователя продолжает работать. + self.assertEqual(api.request('GET', ME, headers={'Authorization': 'Bearer ' + other}).status, 200) + + def test_login_again_after_logout(self): + user = api.new_user('Снова') + api.request('DELETE', SESSIONS + '/current', user=user) + again = api.login(user['username'], user['password']) + self.assertEqual(again.status, 201) + self.assertNotEqual(again.json['token'], user['token']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_validation.py b/student-tests/test_validation.py new file mode 100644 index 0000000..4ce3112 --- /dev/null +++ b/student-tests/test_validation.py @@ -0,0 +1,219 @@ +"""Валидация входных данных: неверный JSON, пустое сообщение, дубликат username, пароль, Unicode.""" +import json +import unittest +import uuid + +import api + +USERS = '/api/v1/users' +PASSWORD = 'Valid-password-123' + + +def registration(**fields): + """Корректное тело регистрации; fields заменяют или добавляют поля.""" + body = {'username': api.unique_username(), 'display_name': 'Имя', 'password': PASSWORD} + body.update(fields) + return body + + +class InvalidJsonTest(unittest.TestCase): + def assert_invalid(self, response): + self.assertEqual(response.status, 400) + self.assertEqual(response.json['error']['code'], 'invalid_request') + + def test_malformed_json_on_register(self): + self.assert_invalid(api.request('POST', USERS, raw=b'{"username": "broken", ')) + + def test_malformed_json_on_message(self): + user = api.new_user() + chat = api.new_chat(user, [user]) + path = '/api/v1/chats/%s/messages' % chat['id'] + for raw in (b'not json', b'{"text": "no closing quote}', b'', b'{"text": }'): + with self.subTest(raw=raw): + self.assert_invalid(api.request('POST', path, user=user, raw=raw)) + # Неудачные запросы ничего не записали в историю. + self.assertEqual(api.pages(path, user, 100), [[]]) + + def test_json_that_is_not_an_object(self): + for raw in (b'[]', b'"text"', b'42', b'null'): + with self.subTest(raw=raw): + self.assert_invalid(api.request('POST', USERS, raw=raw)) + + def test_invalid_utf8(self): + raw = json.dumps(registration(display_name='PLACEHOLDER')).encode().replace(b'PLACEHOLDER', b'\xff\xfe') + self.assert_invalid(api.request('POST', USERS, raw=raw)) + + def test_lone_surrogate_is_rejected_not_crashed(self): + # Всё остальное в теле корректно: ошибка именно в одиночном суррогате. + raw = json.dumps(registration(display_name='PLACEHOLDER')).encode().replace(b'PLACEHOLDER', b'\\ud800') + self.assert_invalid(api.request('POST', USERS, raw=raw)) + + def test_nul_character_is_rejected_not_crashed(self): + # PostgreSQL не хранит символ NUL в text: без проверки это был бы 500. + self.assert_invalid(api.request('POST', USERS, body=registration(display_name='a\x00b'))) + + def test_extra_fields(self): + self.assert_invalid(api.request('POST', USERS, body=registration(is_admin=True))) + + def test_wrong_types(self): + for fields in ({'username': 12345}, {'display_name': ['Имя']}, {'display_name': None}, {'password': 123456789012345}): + with self.subTest(fields=fields): + self.assert_invalid(api.request('POST', USERS, body=registration(**fields))) + + def test_wrong_content_type(self): + raw = json.dumps(registration()).encode() + self.assert_invalid(api.request('POST', USERS, raw=raw, headers={'Content-Type': 'text/plain'})) + + +class UserValidationTest(unittest.TestCase): + def test_duplicate_username_is_409(self): + body = registration(display_name='Первый') + first = api.request('POST', USERS, body=body) + self.assertEqual(first.status, 201) + # Совпадает только username: другие имя и пароль не помогают. + second = api.request('POST', USERS, body=dict(body, display_name='Второй', password='Another-password-456')) + self.assertEqual(second.status, 409) + self.assertEqual(second.json['error']['code'], 'conflict') + # Первый пользователь не изменился: входит со своим паролем и видит своё имя. + session = api.login(body['username'], body['password']) + self.assertEqual(session.json['user'], first.json) + self.assertEqual(api.login(body['username'], 'Another-password-456').status, 401) + + def test_bad_usernames(self): + for username in ('ab', 'a' * 33, 'Alice', 'with space', 'дмитрий', 'dash-name', '', 'name\n', 'colon:name'): + with self.subTest(username=username): + self.assertEqual(api.request('POST', USERS, body=registration(username=username)).status, 400) + + def test_username_length_limits_are_accepted(self): + for length in (3, 32): + with self.subTest(length=length): + response = api.request('POST', USERS, body=registration(username=uuid.uuid4().hex[:length])) + # Трёхсимвольное имя могло остаться от прошлого запуска: 409 тоже значит «имя допустимо». + self.assertIn(response.status, (201, 409)) + + def test_bad_display_names(self): + for display_name in ('', ' ', '\t\n', 'я' * 101): + with self.subTest(display_name=display_name): + self.assertEqual(api.request('POST', USERS, body=registration(display_name=display_name)).status, 400) + + def test_bad_passwords(self): + for password in ('a' * 11, 'я' * 11, 'a' * 129, ' ' * 12, '\t' * 20, 'pass\x00word-123'): + with self.subTest(password=password[:20]): + self.assertEqual(api.request('POST', USERS, body=registration(password=password)).status, 400) + + def test_password_length_limits_are_accepted(self): + # Длина считается в code points: 12 кириллических букв — это 24 байта UTF-8. + for password in ('a' * 12, 'я' * 12, '👋' * 128): + with self.subTest(length=len(password)): + body = registration(password=password) + self.assertEqual(api.request('POST', USERS, body=body).status, 201) + self.assertEqual(api.login(body['username'], password).status, 201) + + def test_missing_fields(self): + for field in ('username', 'display_name', 'password'): + with self.subTest(field=field): + body = registration() + del body[field] + self.assertEqual(api.request('POST', USERS, body=body).status, 400) + + def test_unicode_display_name_is_kept(self): + # Длина считается в code points: 100 эмодзи — это 400 байт, но имя допустимо. + for display_name in ('Дмитрий 👋', '李雷', '👋' * 100): + with self.subTest(display_name=display_name): + response = api.request('POST', USERS, body=registration(display_name=display_name)) + self.assertEqual(response.status, 201) + self.assertEqual(response.json['display_name'], display_name) + + +class MessageValidationTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.user = api.new_user() + cls.chat = api.new_chat(cls.user, [cls.user]) + cls.path = '/api/v1/chats/%s/messages' % cls.chat['id'] + + def test_empty_message_is_400(self): + for text in ('', ' ', '\n\t \r\n', '  '): + with self.subTest(text=text): + response = api.send(self.user, self.chat, text) + self.assertEqual(response.status, 400) + self.assertEqual(response.json['error']['code'], 'invalid_request') + + def test_missing_or_wrong_text(self): + for body in ({}, {'text': None}, {'text': 123}, {'text': ['a']}, {'message': 'a'}): + with self.subTest(body=body): + self.assertEqual(api.request('POST', self.path, user=self.user, body=body).status, 400) + + def test_sender_id_in_body_is_rejected(self): + other = api.new_user() + body = {'text': 'подмена отправителя', 'sender_id': other['id']} + self.assertEqual(api.request('POST', self.path, user=self.user, body=body).status, 400) + + def test_nul_in_text_is_400(self): + self.assertEqual(api.send(self.user, self.chat, 'до\x00после').status, 400) + + def test_text_length_limit(self): + self.assertEqual(api.send(self.user, self.chat, 'я' * 4000).status, 201) + self.assertEqual(api.send(self.user, self.chat, '👋' * 4000).status, 201) + self.assertEqual(api.send(self.user, self.chat, 'я' * 4001).status, 400) + + def test_unicode_text_is_kept(self): + text = 'Привет, мир 👋 — 你好\nвторая строка' + response = api.send(self.user, self.chat, text) + self.assertEqual(response.status, 201) + self.assertEqual(response.json['text'], text) + self.assertEqual(response.json['sender_id'], self.user['id']) + self.assertEqual(response.json['chat_id'], self.chat['id']) + + def test_oversized_body_is_413(self): + response = api.request('POST', self.path, user=self.user, body={'text': 'a' * 70000}) + self.assertEqual(response.status, 413) + self.assertEqual(response.json['error']['code'], 'payload_too_large') + + +class ChatValidationTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.alice = api.new_user() + cls.bob = api.new_user() + + def create(self, title, member_ids): + return api.request('POST', '/api/v1/chats', user=self.alice, body={'title': title, 'member_ids': member_ids}) + + def test_caller_must_be_a_member(self): + self.assertEqual(self.create('Без меня', [self.bob['id']]).status, 400) + + def test_duplicate_members(self): + self.assertEqual(self.create('Дубликат', [self.alice['id'], self.bob['id'], self.bob['id']]).status, 400) + # Тот же UUID в другом регистре — тоже дубликат. + self.assertEqual(self.create('Дубликат', [self.alice['id'], self.alice['id'].upper()]).status, 400) + + def test_unknown_member_rolls_back_the_whole_chat(self): + # Чат вставляется первым, участник — вторым; ошибка на участнике откатывает и сам чат. + title = 'Откат ' + uuid.uuid4().hex[:8] + response = self.create(title, [self.alice['id'], str(uuid.uuid4())]) + self.assertEqual(response.status, 400) + chats = [chat for page in api.pages('/api/v1/chats', self.alice, 100) for chat in page] + self.assertNotIn(title, [chat['title'] for chat in chats]) + + def test_bad_member_ids(self): + for member_ids in ([], 'not a list', [123], ['not-a-uuid'], None): + with self.subTest(member_ids=member_ids): + self.assertEqual(self.create('Чат', member_ids).status, 400) + + def test_bad_titles(self): + for title in ('', ' ', 'я' * 101, 5, 'a\x00b'): + with self.subTest(title=title): + self.assertEqual(self.create(title, [self.alice['id']]).status, 400) + + def test_valid_chat_keeps_member_order(self): + response = self.create('Проект по ВНП', [self.bob['id'], self.alice['id']]) + self.assertEqual(response.status, 201) + self.assertEqual(response.json['title'], 'Проект по ВНП') + self.assertEqual(response.json['member_ids'], [self.bob['id'], self.alice['id']]) + again = api.request('GET', '/api/v1/chats/' + response.json['id'], user=self.bob) + self.assertEqual(again.json, response.json) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_image_process.py b/unit-tests/test_image_process.py new file mode 100644 index 0000000..f599be8 --- /dev/null +++ b/unit-tests/test_image_process.py @@ -0,0 +1,168 @@ +import hashlib +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import Mock, patch +import uuid + +from PIL import Image + +from messenger.image_process import InvalidImage, fit_dimensions, process_image +from messenger.worker import ProcessingError, Worker, retry_delay + + +class ImagePipelineTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.directory = Path(self.temp.name) + self.addCleanup(self.temp.cleanup) + + def source(self, size, mode='RGB', color='red', fmt='PNG', **options): + path = self.directory / 'original' + image = Image.new(mode, size, color) + image.save(path, format=fmt, **options) + image.close() + return path + + def test_all_variants_and_original_checksum(self): + source = self.source((320, 200)) + original = source.read_bytes() + variants = process_image(source, self.directory, 'image/png') + self.assertEqual(source.read_bytes(), original) + self.assertEqual({v['name'] for v in variants}, {'thumbnail', 'crop', 'optimized'}) + for metadata in variants: + filename = {'thumbnail': 'thumbnail.jpg', 'crop': 'crop.jpg', 'optimized': 'optimized.webp'}[metadata['name']] + content = (self.directory / filename).read_bytes() + self.assertEqual(metadata['size_bytes'], len(content)) + self.assertEqual(metadata['sha256'], hashlib.sha256(content).hexdigest()) + with Image.open(self.directory / 'thumbnail.jpg') as image: + self.assertEqual(image.format, 'JPEG') + self.assertEqual(image.size, (256, 160)) + with Image.open(self.directory / 'crop.jpg') as image: + self.assertEqual(image.size, (128, 128)) + with Image.open(self.directory / 'optimized.webp') as image: + self.assertEqual(image.format, 'WEBP') + self.assertEqual(image.size, (320, 200)) + + def test_small_alpha_is_flattened_white_without_upscale(self): + source = self.source((12, 7), 'RGBA', (255, 0, 0, 0)) + process_image(source, self.directory, 'image/png') + with Image.open(self.directory / 'thumbnail.jpg') as image: + self.assertEqual(image.size, (12, 7)) + self.assertTrue(all(channel >= 250 for channel in image.getpixel((1, 1)))) + with Image.open(self.directory / 'optimized.webp') as image: + self.assertEqual(image.size, (12, 7)) + with Image.open(self.directory / 'crop.jpg') as image: + self.assertEqual(image.size, (128, 128)) + + def test_exif_orientation_applied_and_metadata_removed(self): + exif = Image.Exif() + exif[274] = 6 + exif[315] = 'metadata must disappear' + source = self.source((320, 200), fmt='JPEG', exif=exif) + process_image(source, self.directory, 'image/jpeg') + for name in ('thumbnail.jpg', 'crop.jpg', 'optimized.webp'): + with Image.open(self.directory / name) as image: + self.assertFalse(image.getexif()) + self.assertNotIn('icc_profile', image.info) + with Image.open(self.directory / 'thumbnail.jpg') as image: + self.assertEqual(image.size, (160, 256)) + + def test_center_crop_uses_center_pixels(self): + source = self.source((300, 100), color='red') + with Image.open(source) as image: + image.paste('blue', (100, 0, 200, 100)) + image.save(source, format='PNG') + process_image(source, self.directory, 'image/png') + with Image.open(self.directory / 'crop.jpg') as image: + pixel = image.getpixel((64, 64)) + self.assertGreater(pixel[2], 240) + self.assertLess(pixel[0], 15) + + def test_invalid_pixels_and_mime_rejected(self): + source = self.source((11, 10)) + with self.assertRaises(InvalidImage) as error: + process_image(source, self.directory, 'image/png', max_pixels=100) + self.assertEqual(error.exception.code, 'image_too_large') + with self.assertRaises(InvalidImage): + process_image(source, self.directory, 'image/jpeg') + source.write_bytes(b'not an image') + with self.assertRaises(InvalidImage): + process_image(source, self.directory, 'image/png') + + def test_floor_dimensions_and_extreme_aspect_ratio(self): + self.assertEqual(fit_dimensions(321, 200, 256), (256, 159)) + self.assertEqual(fit_dimensions(10_000, 1, 256), (256, 1)) + self.assertEqual(fit_dimensions(1, 10_000, 1280), (1, 1280)) + + def test_decoder_subprocess_under_production_limits(self): + self.source((2000, 1500), 'RGBA', (200, 10, 50, 128)).rename(self.directory / 'source') + worker = Worker(Mock(), Mock(), threading.Event()) + variants = worker._decode({'content_type': 'image/png'}, self.directory) + self.assertEqual({v['name'] for v in variants}, {'thumbnail', 'crop', 'optimized'}) + optimized = next(v for v in variants if v['name'] == 'optimized') + self.assertEqual((optimized['width'], optimized['height']), (1280, 960)) + + +class WorkerDurabilityTests(unittest.TestCase): + def setUp(self): + self.jobs, self.broker = Mock(), Mock() + self.worker = Worker(self.jobs, Mock(), threading.Event()) + self.claim = {'job_id': uuid.uuid4(), 'attachment_id': uuid.uuid4(), 'attempt': 1, + 'run_attempts': 1, 'pipeline_version': 1} + self.payload = {'job_id': str(self.claim['job_id']), + 'attachment_id': str(self.claim['attachment_id']), 'pipeline_version': 1} + self.jobs.claim.return_value = self.claim + + def test_ack_follows_committed_result(self): + with patch.object(self.worker, 'process', return_value=True): + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.broker.ack.assert_called_once_with('1-0') + + def test_lost_fence_is_left_pending(self): + with patch.object(self.worker, 'process', return_value=False): + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.broker.ack.assert_not_called() + + def test_retry_ack_requires_durable_failure_record(self): + self.jobs.fail.return_value = True + with patch.object(self.worker, 'process', side_effect=ProcessingError('dependency_unavailable')): + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.jobs.fail.assert_called_once() + self.assertFalse(self.jobs.fail.call_args.args[2]) + self.broker.ack.assert_called_once_with('1-0') + self.jobs.fail.return_value = False + self.broker.reset_mock() + with patch.object(self.worker, 'process', side_effect=ProcessingError('dependency_unavailable')): + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.broker.ack.assert_not_called() + + def test_permanent_failure_has_no_automatic_processing_retry(self): + self.jobs.fail.return_value = True + with patch.object(self.worker, 'process', side_effect=ProcessingError('invalid_image', permanent=True)): + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.assertTrue(self.jobs.fail.call_args.args[2]) + self.broker.ack.assert_called_once_with('1-0') + + def test_active_redelivery_does_not_ack_and_ready_is_noop(self): + self.jobs.claim.return_value = None + self.jobs.delivery_state.return_value = {'status': 'processing', 'variant_count': 0} + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + self.broker.ack.assert_not_called() + self.jobs.delivery_state.return_value = {'status': 'ready', 'variant_count': 3} + with patch.object(self.worker, 'process') as process: + self.worker.handle(self.broker, '1-0', self.payload, 'consumer') + process.assert_not_called() + self.broker.ack.assert_called_once_with('1-0') + + def test_backoff_bounds_and_cap(self): + rng = Mock() + rng.uniform.side_effect = lambda low, high: high + self.assertEqual(retry_delay(1, rng, base_seconds=1, max_seconds=30), 1) + self.assertEqual(retry_delay(100, rng, base_seconds=1, max_seconds=30), 30) + rng.uniform.assert_called_with(15, 30) + + +if __name__ == '__main__': + unittest.main()