From be223efc8b7438c983078af85e458bea994fd180 Mon Sep 17 00:00:00 2001 From: Polusikkkk Date: Tue, 17 Nov 2026 18:48:50 +0300 Subject: [PATCH] Implement durable image processing pipeline with outbox and recovery --- .env.example | 9 +- Dockerfile | 26 ++ REPORT.md | 309 +++++++++++++++++++-- app/__init__.py | 0 app/auth.py | 108 ++++++++ app/broker.py | 36 +++ app/cache.py | 129 +++++++++ app/cleanup_orphans.py | 208 ++++++++++++++ app/cleanup_staging.py | 198 ++++++++++++++ app/config.py | 12 + app/db.py | 76 ++++++ app/dependencies.py | 48 ++++ app/errors.py | 71 +++++ app/faults.py | 34 +++ app/idempotency.py | 180 +++++++++++++ app/image_pipeline.py | 133 +++++++++ app/images.py | 95 +++++++ app/instance.py | 7 + app/job_db.py | 24 ++ app/job_runner.py | 194 +++++++++++++ app/jobs.py | 90 +++++++ app/main.py | 116 ++++++++ app/migrate.py | 84 ++++++ app/object_locks.py | 12 + app/pagination.py | 128 +++++++++ app/publisher.py | 126 +++++++++ app/replay.py | 188 +++++++++++++ app/responses.py | 40 +++ app/routers/__init__.py | 0 app/routers/attachments.py | 275 +++++++++++++++++++ app/routers/chats.py | 74 +++++ app/routers/health.py | 20 ++ app/routers/messages.py | 90 +++++++ app/routers/sessions.py | 82 ++++++ app/routers/users.py | 59 ++++ app/s3_check.py | 34 +++ app/storage.py | 123 +++++++++ app/store.py | 280 +++++++++++++++++++ app/validation.py | 61 +++++ app/worker.py | 341 +++++++++++++++++++++++ compose.queue.yaml | 112 ++++++++ compose.s3.yaml | 120 +++++++++ compose.yaml | 132 +++++++++ evidence/after_objects.json | 58 ++++ evidence/before_ack.json | 44 +++ evidence/cleanup_tests.txt | 13 + evidence/failed_replay.json | 48 ++++ evidence/failover.txt | 4 + evidence/failover_containers.json | 18 ++ evidence/image_pipeline_tests.txt | 11 + evidence/outbox_gap.json | 34 +++ evidence/persistence.txt | 3 + evidence/queued_restart.json | 38 +++ evidence/restart_containers.json | 163 +++++++++++ evidence/retry_exhaustion.json | 64 +++++ evidence/smoke.txt | 2 + evidence/staging_cleanup.json | 62 +++++ infra/haproxy/haproxy.cfg | 42 +++ migrations/001_initial.sql | 71 +++++ migrations/002_message_idempotency.sql | 22 ++ migrations/003_attachments.sql | 67 +++++ migrations/004_attachment_pipeline.sql | 101 +++++++ migrations/005_fault_injections.sql | 8 + migrations/006_after_objects_fault.sql | 6 + migrations/007_job_replays.sql | 13 + requirements.txt | 9 + scripts/dc | 10 + scripts/restart.sh | 124 +++++++++ scripts/setup_lab5.py | 67 +++++ scripts/setup_s3_secrets.py | 126 +++++++++ scripts/stop-one.sh | 66 +++++ student-tests/after_objects.py | 266 ++++++++++++++++++ student-tests/before_ack.py | 262 ++++++++++++++++++ student-tests/concurrent_idempotency.py | 308 +++++++++++++++++++++ student-tests/failed_replay.py | 294 ++++++++++++++++++++ student-tests/outbox_gap.py | 228 ++++++++++++++++ student-tests/postgres_pause.py | 176 ++++++++++++ student-tests/queued_restart.py | 345 ++++++++++++++++++++++++ student-tests/retry_exhaustion.py | 226 ++++++++++++++++ student-tests/rolling_replacement.py | 284 +++++++++++++++++++ student-tests/staging_cleanup.py | 274 +++++++++++++++++++ student-tests/targeted_failover.py | 318 ++++++++++++++++++++++ student-tests/test_cleanup_staging.py | 147 ++++++++++ student-tests/test_image_pipeline.py | 94 +++++++ 84 files changed, 8982 insertions(+), 18 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/auth.py create mode 100644 app/broker.py create mode 100644 app/cache.py create mode 100644 app/cleanup_orphans.py create mode 100644 app/cleanup_staging.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/dependencies.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/idempotency.py create mode 100644 app/image_pipeline.py create mode 100644 app/images.py create mode 100644 app/instance.py create mode 100644 app/job_db.py create mode 100644 app/job_runner.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/object_locks.py create mode 100644 app/pagination.py create mode 100644 app/publisher.py create mode 100644 app/replay.py create mode 100644 app/responses.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/attachments.py create mode 100644 app/routers/chats.py create mode 100644 app/routers/health.py create mode 100644 app/routers/messages.py create mode 100644 app/routers/sessions.py create mode 100644 app/routers/users.py create mode 100644 app/s3_check.py create mode 100644 app/storage.py create mode 100644 app/store.py create mode 100644 app/validation.py create mode 100644 app/worker.py create mode 100644 compose.queue.yaml create mode 100644 compose.s3.yaml create mode 100644 compose.yaml create mode 100644 evidence/after_objects.json create mode 100644 evidence/before_ack.json create mode 100644 evidence/cleanup_tests.txt create mode 100644 evidence/failed_replay.json create mode 100644 evidence/failover.txt create mode 100644 evidence/failover_containers.json create mode 100644 evidence/image_pipeline_tests.txt create mode 100644 evidence/outbox_gap.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/queued_restart.json create mode 100644 evidence/restart_containers.json create mode 100644 evidence/retry_exhaustion.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/staging_cleanup.json create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_message_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_attachment_pipeline.sql create mode 100644 migrations/005_fault_injections.sql create mode 100644 migrations/006_after_objects_fault.sql create mode 100644 migrations/007_job_replays.sql create mode 100644 requirements.txt create mode 100755 scripts/dc create mode 100755 scripts/restart.sh create mode 100644 scripts/setup_lab5.py create mode 100644 scripts/setup_s3_secrets.py create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/after_objects.py create mode 100644 student-tests/before_ack.py create mode 100644 student-tests/concurrent_idempotency.py create mode 100644 student-tests/failed_replay.py create mode 100644 student-tests/outbox_gap.py create mode 100644 student-tests/postgres_pause.py create mode 100644 student-tests/queued_restart.py create mode 100644 student-tests/retry_exhaustion.py create mode 100644 student-tests/rolling_replacement.py create mode 100644 student-tests/staging_cleanup.py create mode 100644 student-tests/targeted_failover.py create mode 100644 student-tests/test_cleanup_staging.py create mode 100644 student-tests/test_image_pipeline.py diff --git a/.env.example b/.env.example index 119384b5..3f93aba4 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,14 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +# One task per worker process; values other than 1 are rejected. +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 + +# Additional settings used by compose.queue.yaml: +JOB_LEASE_SECONDS=120 +JOB_MEMORY_MB=512 +# Local credentials are generated by scripts/setup_lab5.py. +# See REPORT.md for mappings and settings fixed by the contract. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2f0bbebc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12.10-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +RUN addgroup --system appgroup \ + && adduser --system --ingroup appgroup --no-create-home appuser \ + && chown appuser:appgroup /app + +COPY --chown=appuser:appgroup app ./app +COPY --chown=appuser:appgroup migrations ./migrations + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" || exit 1 + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--no-access-log"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..88b070ab 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,312 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Полина Соколова, БВТ2304. +- Целевая оценка: 4. Исследовательский трек уровня 5 не заявляется. +- Commit/PR: будут указаны при публикации. +- Проверки выполнены 17.11.2026 на локальном учебном стенде. +- Использованы собственные исходники lab4, базовый коммит dcaebee. + Контракт, публичные tests и Makefile взяты из skeleton lab5. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Linux, Docker Engine с Docker Compose plugin, Python 3.9+ +для публичных проверок, make. Приложение работает в Python 3.12.10 +в контейнере. Локальные проверки запускались также на Python 3.11. + +Из корня репозитория: + +```sh +python3 scripts/setup_lab5.py +./scripts/dc config --quiet +./scripts/dc up -d --build +./scripts/dc ps -a +``` + +setup_lab5.py генерирует локальные .env и secrets/, не выводя значения +секретов. При существующей конфигурации скрипт отказывается её перезаписывать. +Повторный обычный запуск не требует повторной генерации секретов. + +scripts/dc явно выбирает проект mtusi2026lab5, .env и три Compose-файла: +compose.yaml, compose.s3.yaml, compose.queue.yaml. + +Перед тестами дождаться HTTP 200: + +```sh +curl --fail http://127.0.0.1:18085/health/ready +make check +make test BASE_URL=http://127.0.0.1:18085 +``` + +Если curl ещё не получил 200, повторить проверку после завершения старта. +Не выполнять тесты через внутренние порты API в обход LB. + +Миграции выполняет одноразовый сервис migrate. +Существующие миграции проверяются по checksum. +s3-check проверяет доступ к приватному bucket перед запуском API/worker. + +Для локальных тестов преобразования: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt 'Pillow==13.0.0' +.venv/bin/python -m unittest discover \ + -s student-tests -p 'test_image_pipeline.py' -v +``` + +Тесты уборщика запускаются с зависимостями приложения: + +```sh +./scripts/dc exec -T api python - \ + < student-tests/test_cleanup_staging.py +``` + +Полная OpenAPI-валидация: + +```sh +make validate PYTHON=.venv/bin/python +``` + +### Версии + +- Python image: 3.12.10-slim-bookworm. +- PostgreSQL: 16.4-bookworm. +- Redis: 7.2.5-bookworm. +- HAProxy: 3.0.5-bookworm. +- SeaweedFS: 4.44. +- RabbitMQ: 4.0.5-management. +- Python-зависимости закреплены в requirements.txt, + включая Pillow 13.0.0 и Pika 1.3.2. + +### Остановка и удаление данных + +Безопасная остановка без удаления данных: + +```sh +./scripts/dc stop +``` + +Повторный запуск: + +```sh +./scripts/dc up -d +``` + +Только для намеренного полного удаления учебных данных lab5: + +```sh +./scripts/dc down --volumes +``` + +Последняя команда удаляет volumes проекта, включая БД, S3 и очередь. +Не использовать её в проверках сохранности и отказов. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +клиент → HAProxy → две API-реплики + ├→ PostgreSQL: метаданные + job + outbox + └→ приватный S3 staging + +outbox publisher → RabbitMQ → worker → дочерний job_runner + ├→ оригинал и варианты в S3 + └→ COMMIT результата в PostgreSQL +``` + +PostgreSQL — источник истины для пользователей, сессий, чатов, +сообщений, метаданных и логических заданий. Redis используется +перенесённой реализацией кэша; не является единственным хранилищем сессий. + +API принимает сырые bytes PNG/JPEG и проверяет входное изображение. +До 202 оригинал записан в приватный S3-prefix staging/, а вложение, +job и outbox закоммичены одной транзакцией PostgreSQL. +Это не общая транзакция PostgreSQL/S3: запись без успешного COMMIT +может оставить непривязанный staging-объект. Его удаляет уборщик. + +API возвращает снимок queued. Он не создаёт thumbnail синхронно. +Валидация исходного изображения остаётся синхронной частью upload. + +Publisher читает outbox, публикует persistent-сообщение в durable +очередь attachment.process.v1 с mandatory и publisher confirms. +published_at фиксируется после подтверждения брокера. +Crash после publish до фиксации published_at допускает дубль. + +В сообщении только идентификаторы и служебные поля по jobs.schema.json. +Credentials, bytes, клиентский filename и произвольный URL туда не входят. +Worker получает доверенные storage metadata из PostgreSQL. + +Worker захватывает job с lease token. Преобразование и операции одной +попытки выполняются в дочернем процессе с внешним wall-clock timeout. +Результат публикуется транзакционно после записи всех объектов и проверки +текущего lease. Ack отправляется после устойчивого результата либо +устойчивой фиксации retry/failed. + +Доставка at least once, не exactly once. +Повтор ready-job не откатывает состояние и не запускает преобразование. +Для текущего pipeline_version=1 ключи вариантов детерминированы. +Ключ метаданных варианта: attachment_id + pipeline_version + name. +Исследование конкурирующих версий pipeline не проводилось. + +### Варианты изображений + +- Original возвращается побайтно без перекодирования. +- Thumbnail: JPEG, вписан в 256×256, без увеличения, округление вниз. +- Crop: JPEG, центрированный квадрат 128×128, увеличение допускается. +- Optimized: WebP, вписан в 1280×1280 без увеличения. +- EXIF orientation применяется перед преобразованиями. +- Alpha вариантов компонуется на белый фон. +- JPEG quality=85, WebP quality=82, method=4. +- Метаданные EXIF/ICC/XMP исходника не переносятся в варианты. +- WebP не обязан быть меньше каждого исходного файла. + +### Retry, ограничения и failed + +- По умолчанию 5 попыток. +- Задержка retry: uniform(0.5, min(30, 2**min(attempt, 5))) секунд. +- InvalidImage и пойманный MemoryError — постоянные ошибки. +- Прочие ошибки дочернего исполнения и timeout — временные; + после исчерпания попыток — failed/retry_exhausted. +- Реестр failed находится в attachment_jobs. +- Один worker обрабатывает одну задачу одновременно. +- Worker задаёт prefetch_count=1 и использует basic_get по одной доставке. +- Фактическое ограничение параллелизма обеспечивается кодом worker; + basic_qos не заявляется отдельной гарантией для polling basic_get. +- Upload: максимум 10 485 760 bytes и 40 000 000 пикселей. +- По умолчанию timeout попытки 60 секунд, lease 120 секунд. +- Дочерний процесс: RLIMIT_AS 512 MiB и RLIMIT_CPU. + RLIMIT_AS ограничивает адресное пространство, не точный RSS. +- Контейнер worker: mem_limit 768m, cpus=1. +- Исчерпание памяти или аварийный сигнал, не пойманный как MemoryError, + могут обрабатываться как временная ошибка до исчерпания попыток. +- Верхние границы времени и памяти установлены в коде/Compose; + отдельный эксперимент на предельных изображениях не проводился. + +### Операторский replay + +```sh +./scripts/dc exec -T worker python -m app.replay --list +``` + +Для failed-job сначала dry-run: + +```sh +./scripts/dc exec -T worker python -m app.replay \ + --job-id UUID --actor operator --reason "Причина устранена" +``` + +После проверки — та же команда с --apply. +Apply проверяет checksum staging, сохраняет job_id, создаёт аудит и +новую попытку/outbox одной транзакцией. По умолчанию выдаётся бюджет +пяти дополнительных попыток. Счётчик попыток не обнуляется. +Ready/queued/processing или устаревшая работа дают noop. +Dry-run показывает план, но не проверяет чтение staging; это делает apply. + +### Retention staging + +Worker удаляет staging после COMMIT результата. +Периодический cleanup проходит раз в час с grace=86400 секунд: + +- queued/processing не удаляются по возрасту; +- ready-остатки и непривязанные объекты удаляются после grace; +- failed сохраняется минимум grace после изменения состояния + и минимум grace после последней записи объекта; +- после удаления failed-staging replay недоступен; +- метаданные failed и аудит автоматически не удаляются. + +Уборщик использует ту же advisory-блокировку ключа, что upload, +и блокировку строки вложения для согласования с replay. +Originals и variants этот уборщик не удаляет. +Нулевой/короткий grace разрешён только явно и для одного attachment_id. + +### Соответствие конфигурации + +scripts/setup_lab5.py, а не копирование .env.example, создаёт рабочую +локальную конфигурацию. Исходный .env.example содержит также предложения +логического интерфейса skeleton. + +- POSTGRES_PASSWORD, RABBITMQ_PASSWORD — локальные значения из .env. +- SESSION_TTL_SECONDS, USER_CACHE_TTL_SECONDS — передаются API. +- S3_*_FILE и BROKER_URL_FILE — пути подключённых Compose secrets. +- JOB_MAX_ATTEMPTS — бюджет новых job и операторского replay. +- JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS, JOB_MEMORY_MB — настройки worker. +- WORKER_CONCURRENCY — поддерживается только 1; другое значение отклоняется. +- STAGING_RETENTION_SECONDS — срок фоновой уборки. +- Две API-реплики заданы в compose.yaml, не через API_REPLICAS. +- MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS закреплены кодом по контракту. +- DATABASE_URL_FILE, CACHE_URL_FILE, SESSION_STORE и + IDEMPOTENCY_TTL_SECONDS из предложений skeleton не являются + реализованными переключателями этой конфигурации. +- Подключение PostgreSQL использует PGHOST/PGPORT/PGDATABASE/PGUSER/PGPASSWORD. + +Readiness API проверяет PostgreSQL и S3. Broker/worker не входят +в синхронный readiness: принятая работа может ждать в durable outbox. +Это не означает неограниченную ёмкость хранилищ. +Liveness проверяет только жизнь API-процесса. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда | Наблюдение | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный API | make test BASE_URL=http://127.0.0.1:18085 | PASS, 202 и настоящий thumbnail | evidence/smoke.txt | +| Worker остановлен, рестарт API/broker | .venv/bin/python student-tests/queued_restart.py | То же вложение стало ready, оригинал и варианты доступны | evidence/queued_restart.json | +| Разрыв COMMIT/publish | .venv/bin/python student-tests/outbox_gap.py | После SIGKILL API существующий outbox опубликован без новой job | evidence/outbox_gap.json | +| После COMMIT результата до ack | .venv/bin/python student-tests/before_ack.py | Redelivered=True, attempt=1, варианты=3, объекты=4 | evidence/before_ack.json | +| После записи S3 до COMMIT результата | .venv/bin/python student-tests/after_objects.py | Сначала processing и 0 строк вариантов; после lease та же job ready на попытке 2 | evidence/after_objects.json | +| Failed и replay | .venv/bin/python student-tests/failed_replay.py | Проверены отказ при повреждении, dry-run, аудит, replay и ready-noop | evidence/failed_replay.json | +| Исчерпание retry | .venv/bin/python student-tests/retry_exhaustion.py | 5 опубликованных попыток, failed/retry_exhausted, без вариантов | evidence/retry_exhaustion.json | +| Уборка staging | .venv/bin/python student-tests/staging_cleanup.py | Активный queued сохранён, orphan и ready-остаток удалены | evidence/staging_cleanup.json | +| Преобразования | unittest test_image_pipeline.py | 6 тестов PASS | evidence/image_pipeline_tests.txt | +| Защитные ветки cleanup | API python через stdin с test_cleanup_staging.py | 8 тестов PASS, подставные БД/S3 | evidence/cleanup_tests.txt | +| Сохранность стенда | make test-persistence с scripts/restart.sh | Данные и прежняя сессия сохранены, хранилища перезапущены с прежними volumes | evidence/persistence.txt; evidence/restart_containers.json | +| Отказ API | make test-failover с scripts/stop-one.sh | 2 реплики до отказа, 130 успешных записей, 2 ошибки, восстановление 2.089 с | evidence/failover.txt; evidence/failover_containers.json | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +PASS в таблице относится к выполненному сценарию, не к универсальной +гарантии при любых отказах. Для воспроизведения failure-тестов нужны +работающие publisher и worker и один worker-контейнер. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Исследовательская нагрузка уровня 5 не проводилась. +Failover-цифры — результат публичного сценария отказа, не benchmark +максимального throughput и не SLA. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Fault injection before_ack и after_objects включается только явной +записью оператора для конкретной job в pipeline_fault_injections. +Воздействие одноразовое, с consumed_at. Тесты используют exit code 75. +Эти точки оставлены для локальной учебной защиты. + +В after_objects проверено восстановление по реальному lease=120 секунд. +В retry_exhaustion проверены сохранённые границы задержек, но не +статистическое распределение jitter и не точность времени доставки. + +Исправлена настройка healthcheck RabbitMQ: две диагностические команды +не укладывались в общий timeout=5s. Проверки вручную завершались успешно. +Установлены interval=20s, timeout=30s, start_period=40s, retries=3. +Проверка не отключена: проверяются работа приложения RabbitMQ +и listener 5672. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host, единичные PostgreSQL, S3, RabbitMQ и LB: + отказ хоста и HA этих компонентов не обеспечиваются. +- HTTP допустим только для локальных синтетических данных. +- Compose file-backed secrets не заменяют промышленное secret-хранилище. +- Администратор учебного брокера/S3 имеет широкие права. +- Очередь и staging не имеют реализованной квоты принятой работы. +- Отдельный эксперимент перегрузки, RSS и p95 до ready не проводился. +- Нет доказательства exactly-once доставки. +- Несколько конкурирующих pipeline_version не исследовались. +- Подставные тесты cleanup не доказывают реальную конкурентную блокировку. +- Удаление непривязанных конечных originals/variants не реализовано + новым staging-уборщиком. +- Финальное выполнение всех тестов и публикация PR указываются отдельно. + +## Источники и библиотеки + +- Контракт и README skeleton lab5. +- RabbitMQ: Reliability Guide; Consumer Acknowledgements and Publisher Confirms. +- Pika: BlockingConnection, delivery confirmations, mandatory publishing. +- Pillow: ImageOps, EXIF transpose, resize и сохранение JPEG/WebP. +- Python: subprocess и resource. +- PostgreSQL: транзакции, row/advisory locks, ограничения. +- Docker Compose: services, secrets, healthchecks, persistence volumes. +- Полный список закреплённых Python-библиотек: requirements.txt. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..3db7b8a2 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import base64 +import binascii +import hashlib +import re +import secrets + +from argon2 import PasswordHasher, Type +from argon2.exceptions import VerificationError +from starlette.concurrency import run_in_threadpool + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}", re.ASCII) + +password_hasher = PasswordHasher( + time_cost=3, + memory_cost=65536, + parallelism=4, + hash_len=32, + salt_len=16, + type=Type.ID, +) + +DUMMY_PASSWORD_HASH = password_hasher.hash( + secrets.token_urlsafe(32) +) + + +async def hash_password(password: str) -> str: + return await run_in_threadpool( + password_hasher.hash, + password, + ) + + +async def verify_password( + password_hash: str, + password: str, +) -> bool: + try: + return await run_in_threadpool( + password_hasher.verify, + password_hash, + password, + ) + except VerificationError: + return False + + +def parse_basic_credentials( + authorization: str | None, +) -> tuple[str, str] | None: + if authorization is None or len(authorization) > 2048: + return None + + parts = authorization.split() + + if len(parts) != 2 or parts[0].lower() != "basic": + return None + + try: + decoded = base64.b64decode( + parts[1], + validate=True, + ).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + return None + + username, separator, password = decoded.partition(":") + + if not separator: + return None + + if USERNAME_RE.fullmatch(username) is None: + return None + + if not 12 <= len(password) <= 128: + return None + + return username, password + + +def parse_bearer_token( + authorization: str | None, +) -> str | None: + if authorization is None or len(authorization) > 2048: + return None + + parts = authorization.split() + + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + + token = parts[1] + + if re.fullmatch(r"[A-Za-z0-9_-]{43}", token) is None: + return None + + return token + + +def generate_token() -> str: + return secrets.token_urlsafe(32) + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("ascii")).hexdigest() \ No newline at end of file diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..61bb77ce --- /dev/null +++ b/app/broker.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pika + + +QUEUE_NAME = "attachment.process.v1" + + +def connect_broker(): + url = Path(os.environ["BROKER_URL_FILE"]).read_text().strip() + if not url: + raise ValueError("Empty broker URL file") + + parameters = pika.URLParameters(url) + parameters.heartbeat = 15 + parameters.blocked_connection_timeout = 5 + parameters.socket_timeout = 5 + parameters.stack_timeout = 10 + parameters.connection_attempts = 1 + + connection = pika.BlockingConnection(parameters) + try: + channel = connection.channel() + channel.queue_declare( + queue=QUEUE_NAME, + durable=True, + exclusive=False, + auto_delete=False, + ) + return connection, channel + except Exception: + connection.close() + raise diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..acd7d21a --- /dev/null +++ b/app/cache.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import json +import logging +import os +from datetime import datetime, timezone +from uuid import UUID + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from app.store import store + + +logger = logging.getLogger("messenger.cache") + +CACHE_TTL_SECONDS = int( + os.getenv("USER_CACHE_TTL_SECONDS", "60") +) + +if not 1 <= CACHE_TTL_SECONDS <= 3600: + raise ValueError( + "USER_CACHE_TTL_SECONDS must be between 1 and 3600" + ) + + +redis_client = Redis( + host=os.getenv("REDIS_HOST", "redis"), + port=int(os.getenv("REDIS_PORT", "6379")), + decode_responses=True, + socket_connect_timeout=0.2, + socket_timeout=0.2, + retry_on_timeout=False, + max_connections=10, +) + + +def serialize_user(user: dict) -> str: + created_at = user["created_at"] + + return json.dumps( + { + "id": str(user["id"]), + "username": user["username"], + "display_name": user["display_name"], + "created_at": created_at.astimezone( + timezone.utc + ).isoformat().replace("+00:00", "Z"), + }, + ensure_ascii=False, + separators=(",", ":"), + ) + + +def deserialize_user( + raw: str, + expected_id: UUID, +) -> dict: + data = json.loads(raw) + + if not isinstance(data, dict): + raise ValueError("Invalid cached user") + + expected_fields = { + "id", + "username", + "display_name", + "created_at", + } + + if set(data) != expected_fields: + raise ValueError("Invalid cached user fields") + + if not all(isinstance(value, str) for value in data.values()): + raise ValueError("Invalid cached user types") + + user_id = UUID(data["id"]) + + if user_id != expected_id: + raise ValueError("Cached user ID mismatch") + + created_at = datetime.fromisoformat( + data["created_at"].replace("Z", "+00:00") + ) + + if created_at.tzinfo is None: + raise ValueError("Cached timestamp has no timezone") + + return { + "id": user_id, + "username": data["username"], + "display_name": data["display_name"], + "created_at": created_at, + } + + +async def get_cached_user(user_id: UUID) -> dict | None: + key = f"user:v1:{user_id}" + + try: + raw = await redis_client.get(key) + + if raw is not None: + return deserialize_user(raw, user_id) + + except RedisError: + logger.warning("User cache read unavailable") + except (ValueError, TypeError, OverflowError): + logger.warning("Invalid user cache entry ignored") + + user = await store.get_user(user_id) + + if user is None: + return None + + try: + await redis_client.set( + key, + serialize_user(user), + ex=CACHE_TTL_SECONDS, + ) + except RedisError: + logger.warning("User cache write unavailable") + + return user + + +async def close_cache() -> None: + await redis_client.aclose() \ No newline at end of file diff --git a/app/cleanup_orphans.py b/app/cleanup_orphans.py new file mode 100644 index 00000000..519caf39 --- /dev/null +++ b/app/cleanup_orphans.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timedelta, timezone +import json +import os +import sys +import time +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row +from botocore.exceptions import ClientError + +from app.object_locks import object_lock_id +from app.storage import bucket_name, get_client + + +def managed_key(key: str) -> bool: + prefix = "originals/" + if not key.startswith(prefix): + return False + + suffix = key[len(prefix):] + try: + return str(UUID(suffix)) == suffix + except ValueError: + return False + + +def database_connection(): + return psycopg.connect( + host=os.getenv("PGHOST", "postgres"), + port=int(os.getenv("PGPORT", "5432")), + dbname=os.getenv("PGDATABASE", "messenger"), + user=os.getenv("PGUSER", "messenger"), + password=os.environ["PGPASSWORD"], + connect_timeout=3, + options=( + "-c statement_timeout=3000 " + "-c lock_timeout=2000 " + "-c idle_in_transaction_session_timeout=10000" + ), + autocommit=True, + row_factory=dict_row, + ) + + +def sweep(*, apply: bool, grace_seconds: int) -> dict: + cutoff = datetime.now(timezone.utc) - timedelta( + seconds=grace_seconds + ) + counts = Counter({ + "scanned": 0, + "kept": 0, + "fresh": 0, + "busy": 0, + "ignored": 0, + "gone": 0, + "candidates": 0, + "deleted": 0, + }) + + with database_connection() as conn: + client = get_client() + bucket = bucket_name() + pages = client.get_paginator("list_objects_v2") + + for page in pages.paginate( + Bucket=bucket, + Prefix="originals/", + PaginationConfig={"PageSize": 200}, + ): + for item in page.get("Contents", []): + counts["scanned"] += 1 + key = item["Key"] + + if not managed_key(key): + counts["ignored"] += 1 + continue + + if item["LastModified"] >= cutoff: + counts["fresh"] += 1 + continue + + with conn.transaction(): + conn.execute( + "SET LOCAL " + "idle_in_transaction_session_timeout = '60s'" + ) + + locked = conn.execute( + "SELECT pg_try_advisory_xact_lock(%s) AS locked", + (object_lock_id(bucket, key),), + ).fetchone()["locked"] + + if not locked: + counts["busy"] += 1 + continue + + metadata = conn.execute( + """ + SELECT 1 + FROM attachments + WHERE object_key = %s + """, + (key,), + ).fetchone() + + if metadata is not None: + counts["kept"] += 1 + continue + + try: + current = client.head_object( + Bucket=bucket, + Key=key, + ) + except ClientError as exc: + status = exc.response.get( + "ResponseMetadata", {} + ).get("HTTPStatusCode") + if status == 404: + counts["gone"] += 1 + continue + raise + + if current["LastModified"] >= cutoff: + counts["fresh"] += 1 + continue + + counts["candidates"] += 1 + print( + json.dumps({ + "action": "delete" if apply else "would_delete", + "key": key, + }), + flush=True, + ) + + if apply: + client.delete_object( + Bucket=bucket, + Key=key, + ) + counts["deleted"] += 1 + + return { + "mode": "apply" if apply else "dry-run", + "grace_seconds": grace_seconds, + **dict(counts), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true") + parser.add_argument("--grace-seconds", type=int, default=86400) + parser.add_argument("--allow-short-grace", action="store_true") + parser.add_argument("--interval-seconds", type=int, default=0) + args = parser.parse_args() + + if args.grace_seconds < 60: + parser.error("grace must be at least 60 seconds") + + if args.grace_seconds < 3600 and not args.allow_short_grace: + parser.error( + "short test grace requires --allow-short-grace" + ) + + if args.interval_seconds != 0 and args.interval_seconds < 60: + parser.error("interval must be zero or at least 60 seconds") + + if args.grace_seconds < 3600: + print( + "WARNING: short grace is for controlled local tests", + file=sys.stderr, + flush=True, + ) + + while True: + try: + result = sweep( + apply=args.apply, + grace_seconds=args.grace_seconds, + ) + except Exception as exc: + print( + json.dumps({ + "status": "failed", + "error_type": type(exc).__name__, + }), + file=sys.stderr, + flush=True, + ) + return 1 + + print(json.dumps(result, sort_keys=True), flush=True) + + if args.interval_seconds == 0: + return 0 + + time.sleep(args.interval_seconds) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/cleanup_staging.py b/app/cleanup_staging.py new file mode 100644 index 00000000..208edbf5 --- /dev/null +++ b/app/cleanup_staging.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timedelta, timezone +import json +import os +import re +import time +from uuid import UUID + +from botocore.exceptions import ClientError + +from app.job_db import connect_db +from app.object_locks import object_lock_id +from app.storage import bucket_name, get_client + + +KEY_PATTERN = re.compile( + r"staging/([0-9a-f-]{36})/v([1-9][0-9]*)" +) + + +def managed_key(key): + match = KEY_PATTERN.fullmatch(key) + if match is None: + return False + try: + return str(UUID(match.group(1))) == match.group(1) + except ValueError: + return False + + +def sweep(*, apply, grace_seconds, attachment_id=None): + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + counts = Counter({ + "scanned": 0, + "ignored": 0, + "fresh": 0, + "busy": 0, + "active": 0, + "gone": 0, + "candidates": 0, + "deleted": 0, + }) + + bucket = bucket_name() + client = get_client() + prefix = ( + f"staging/{attachment_id}/" + if attachment_id is not None + else "staging/" + ) + + with connect_db() as db: + for page in client.get_paginator("list_objects_v2").paginate( + Bucket=bucket, + Prefix=prefix, + PaginationConfig={"PageSize": 200}, + ): + for item in page.get("Contents", []): + counts["scanned"] += 1 + key = item["Key"] + + if not managed_key(key): + counts["ignored"] += 1 + continue + + if item["LastModified"] >= cutoff: + counts["fresh"] += 1 + continue + + with db.transaction(): + db.execute( + "SET LOCAL idle_in_transaction_session_timeout = '60s'" + ) + locked = db.execute( + "SELECT pg_try_advisory_xact_lock(%s) AS locked", + (object_lock_id(bucket, key),), + ).fetchone()["locked"] + + if not locked: + counts["busy"] += 1 + continue + + row = db.execute( + """ + SELECT id, status, updated_at + FROM attachments + WHERE staging_key = %s + FOR UPDATE + """, + (key,), + ).fetchone() + + if row is not None: + if row["status"] in ("queued", "processing"): + counts["active"] += 1 + continue + + if row["status"] == "failed" and row["updated_at"] >= cutoff: + counts["fresh"] += 1 + continue + + try: + current = client.head_object(Bucket=bucket, Key=key) + except ClientError as exc: + status = exc.response.get( + "ResponseMetadata", {} + ).get("HTTPStatusCode") + if status == 404: + counts["gone"] += 1 + continue + raise + + if current["LastModified"] >= cutoff: + counts["fresh"] += 1 + continue + + counts["candidates"] += 1 + + if apply: + client.delete_object(Bucket=bucket, Key=key) + + if row is not None: + db.execute( + """ + UPDATE attachments + SET staging_key = NULL + WHERE id = %s + AND staging_key = %s + AND status IN ('ready', 'failed') + """, + (row["id"], key), + ) + + counts["deleted"] += 1 + + return { + "mode": "apply" if apply else "dry-run", + "grace_seconds": grace_seconds, + "attachment_id": ( + str(attachment_id) if attachment_id is not None else None + ), + **dict(counts), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true") + parser.add_argument( + "--grace-seconds", + type=int, + default=int(os.getenv("STAGING_RETENTION_SECONDS", "86400")), + ) + parser.add_argument("--attachment-id", type=UUID) + parser.add_argument("--allow-short-grace", action="store_true") + parser.add_argument("--interval-seconds", type=int, default=0) + args = parser.parse_args() + + if args.grace_seconds < 0: + parser.error("grace must not be negative") + + if args.grace_seconds < 3600 and ( + not args.allow_short_grace or args.attachment_id is None + ): + parser.error( + "short grace requires --allow-short-grace " + "and a specific --attachment-id" + ) + + if args.interval_seconds != 0 and args.interval_seconds < 60: + parser.error("interval must be zero or at least 60 seconds") + + while True: + try: + result = sweep( + apply=args.apply, + grace_seconds=args.grace_seconds, + attachment_id=args.attachment_id, + ) + except Exception as exc: + print(json.dumps({ + "result": "error", + "error_type": type(exc).__name__, + }), flush=True) + raise SystemExit(1) + + print(json.dumps(result), flush=True) + + if args.interval_seconds == 0: + return + time.sleep(args.interval_seconds) + + +if __name__ == "__main__": + main() diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..477e8bbc --- /dev/null +++ b/app/config.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import os + + +SESSION_TTL_SECONDS = int( + os.getenv("SESSION_TTL_SECONDS", "3600") +) + + +if SESSION_TTL_SECONDS < 1: + raise ValueError("SESSION_TTL_SECONDS must be positive") diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..8aedf8d2 --- /dev/null +++ b/app/db.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio +import os + +from psycopg import AsyncConnection, OperationalError, waiting +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool + + +DB_IO_TIMEOUT_SECONDS = float( + os.getenv("DB_IO_TIMEOUT_SECONDS", "3") +) + +if DB_IO_TIMEOUT_SECONDS <= 0: + raise ValueError("DB_IO_TIMEOUT_SECONDS must be positive") + + +class DeadlineConnection(AsyncConnection): + async def wait(self, gen, interval=0.1): + try: + async with asyncio.timeout(DB_IO_TIMEOUT_SECONDS): + return await waiting.wait_async( + gen, + self.pgconn.socket, + interval=interval, + ) + + except TimeoutError: + await self.close() + raise OperationalError( + "PostgreSQL response deadline exceeded" + ) from None + + except asyncio.CancelledError: + await self.close() + raise + + +pool = AsyncConnectionPool( + conninfo="", + connection_class=DeadlineConnection, + kwargs={ + "host": os.getenv("PGHOST", "postgres"), + "port": int(os.getenv("PGPORT", "5432")), + "dbname": os.getenv("PGDATABASE", "messenger"), + "user": os.getenv("PGUSER", "messenger"), + "password": os.environ.get("PGPASSWORD", ""), + "connect_timeout": 3, + "options": ( + "-c statement_timeout=3000 " + "-c lock_timeout=2000 " + "-c idle_in_transaction_session_timeout=10000" + ), + "row_factory": dict_row, + }, + min_size=1, + max_size=5, + timeout=3, + max_waiting=20, + open=False, + check=AsyncConnectionPool.check_connection, +) + + +async def open_db() -> None: + await pool.open(wait=True, timeout=10) + + +async def close_db() -> None: + await pool.close() + + +async def check_db() -> None: + async with pool.connection() as conn: + await conn.execute("SELECT 1") diff --git a/app/dependencies.py b/app/dependencies.py new file mode 100644 index 00000000..59efb145 --- /dev/null +++ b/app/dependencies.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import Request + +from app.auth import hash_token, parse_bearer_token +from app.cache import get_cached_user +from app.errors import not_found, unauthorized +from app.store import store +from app.validation import parse_uuid + + +async def current_user(request: Request) -> tuple[dict, str]: + token = parse_bearer_token( + request.headers.get("authorization") + ) + + if token is None: + raise unauthorized() + + token_digest = hash_token(token) + + user_id = await store.get_session_user_id(token_digest) + + if user_id is None: + raise unauthorized() + + user = await get_cached_user(user_id) + + if user is None: + raise unauthorized() + + return user, token_digest + + +async def accessible_chat( + raw_chat_id: str, + user_id: UUID, +) -> tuple[UUID, dict]: + chat_id = parse_uuid(raw_chat_id) + + chat = await store.get_accessible_chat(chat_id, user_id) + + if chat is None: + raise not_found() + + return chat_id, chat diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..adc2fb83 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from uuid import uuid4 + +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + challenge: str | None = None, + ): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.challenge = challenge + + +def invalid(message: str = "Некорректный запрос") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + if scheme == "Basic": + challenge = 'Basic realm="messenger", charset="UTF-8"' + else: + challenge = 'Bearer realm="messenger"' + + return ApiError( + 401, + "unauthorized", + "Недействительная аутентификация", + challenge, + ) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def error_response( + request: Request, + error: ApiError, +) -> JSONResponse: + request_id = getattr( + request.state, + "request_id", + str(uuid4()), + ) + + headers = {"X-Request-Id": request_id} + + if error.challenge is not None: + headers["WWW-Authenticate"] = error.challenge + + return JSONResponse( + status_code=error.status, + content={ + "error": { + "code": error.code, + "message": error.message, + "request_id": request_id, + } + }, + headers=headers, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 00000000..c99aa2a8 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import logging +import os + +from app.job_db import connect_db + + +logger = logging.getLogger("messenger.faults") + + +def trip_fault(job_id, stage: str) -> None: + with connect_db() as db: + row = db.execute( + """ + UPDATE pipeline_fault_injections + SET consumed_at = clock_timestamp() + WHERE job_id = %s + AND stage = %s + AND consumed_at IS NULL + RETURNING job_id + """, + (job_id, stage), + ).fetchone() + + if row is None: + return + + logger.warning( + "fault_injection job_id=%s stage=%s exit_code=75", + job_id, + stage, + ) + os._exit(75) diff --git a/app/idempotency.py b/app/idempotency.py new file mode 100644 index 00000000..eaaec334 --- /dev/null +++ b/app/idempotency.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import hashlib +import json +from uuid import UUID, uuid4 + +from app.db import pool +from app.errors import ApiError, invalid, not_found + + +async def create_message_once( + chat_id: UUID, + sender_id: UUID, + text: str, + key: str, + attachment_ids: list[UUID] | None = None, +) -> dict: + attachment_ids = list( + attachment_ids if attachment_ids is not None else [] + ) + + if ( + len(attachment_ids) > 10 + or len(set(attachment_ids)) != len(attachment_ids) + ): + raise invalid("Некорректный список вложений") + + if not isinstance(text, str) or len(text) > 4000: + raise invalid("Некорректный текст сообщения") + + if not text.strip() and not attachment_ids: + raise invalid("Сообщение должно содержать текст или вложение") + + scope = json.dumps( + [ + str(sender_id), + "POST", + f"/api/v1/chats/{chat_id}/messages", + key, + ], + separators=(",", ":"), + ) + + lock_id = int.from_bytes( + hashlib.sha256(scope.encode("utf-8")).digest()[:8], + byteorder="big", + signed=True, + ) + + async with pool.connection() as conn: + await conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (lock_id,), + ) + + cursor = await conn.execute( + """ + SELECT i.request_text, + i.request_attachment_ids, + m.id, m.chat_id, m.sender_id, + m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id + FROM message_attachments AS ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) AS attachment_ids + FROM message_idempotency AS i + JOIN messages AS m ON m.id = i.message_id + WHERE i.user_id = %s + AND i.chat_id = %s + AND i.http_method = 'POST' + AND i.idempotency_key = %s + """, + (sender_id, chat_id, key), + ) + previous = await cursor.fetchone() + + if previous is not None: + if ( + previous["request_text"] != text + or previous["request_attachment_ids"] != attachment_ids + ): + raise ApiError( + 409, + "conflict", + "Ключ уже использован с другим сообщением", + ) + + message = { + field: previous[field] + for field in ( + "id", + "chat_id", + "sender_id", + "text", + "created_at", + "attachment_ids", + ) + } + else: + if attachment_ids: + cursor = await conn.execute( + """ + SELECT id, status + FROM attachments + WHERE id = ANY(%s::uuid[]) + AND chat_id = %s + ORDER BY id + FOR SHARE + """, + (attachment_ids, chat_id), + ) + available = await cursor.fetchall() + + if len(available) != len(attachment_ids): + raise not_found() + + if any(row["status"] == "failed" for row in available): + raise ApiError( + 409, + "conflict", + "Нельзя прикрепить вложение с ошибкой обработки", + ) + + cursor = await conn.execute( + """ + INSERT INTO messages ( + id, chat_id, sender_id, text + ) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, + text, created_at + """, + (uuid4(), chat_id, sender_id, text), + ) + message = await cursor.fetchone() + + if attachment_ids: + async with conn.cursor() as cursor: + await cursor.executemany( + """ + INSERT INTO message_attachments ( + message_id, attachment_id, position + ) + VALUES (%s, %s, %s) + """, + [ + (message["id"], attachment_id, position) + for position, attachment_id + in enumerate(attachment_ids) + ], + ) + + await conn.execute( + """ + INSERT INTO message_idempotency ( + user_id, + chat_id, + http_method, + idempotency_key, + request_text, + request_attachment_ids, + message_id + ) + VALUES (%s, %s, 'POST', %s, %s, %s::uuid[], %s) + """, + ( + sender_id, + chat_id, + key, + text, + attachment_ids, + message["id"], + ), + ) + + message["attachment_ids"] = attachment_ids + + return message diff --git a/app/image_pipeline.py b/app/image_pipeline.py new file mode 100644 index 00000000..f81f7379 --- /dev/null +++ b/app/image_pipeline.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import hashlib +import io +import warnings + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10_485_760 +MAX_PIXELS = 40_000_000 +JPEG_QUALITY = 85 +WEBP_QUALITY = 82 + + +class InvalidImage(ValueError): + pass + + +def fitted_size(width: int, height: int, limit: int) -> tuple[int, int]: + longest = max(width, height) + if longest <= limit: + return width, height + return ( + max(1, width * limit // longest), + max(1, height * limit // longest), + ) + + +def encode_variant(name: str, image: Image.Image, fmt: str) -> dict: + image.info.clear() + output = io.BytesIO() + + if fmt == "JPEG": + image.save( + output, + format="JPEG", + quality=JPEG_QUALITY, + optimize=True, + exif=b"", + icc_profile=None, + ) + content_type = "image/jpeg" + else: + image.save( + output, + format="WEBP", + quality=WEBP_QUALITY, + method=4, + exif=b"", + icc_profile=b"", + xmp=b"", + ) + content_type = "image/webp" + + data = output.getvalue() + return { + "name": name, + "content_type": content_type, + "size_bytes": len(data), + "width": image.width, + "height": image.height, + "sha256": hashlib.sha256(data).hexdigest(), + "data": data, + } + + +def render_variants(data: bytes) -> list[dict]: + if not 1 <= len(data) <= MAX_BYTES: + raise InvalidImage("invalid_size") + + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + + with Image.open(io.BytesIO(data)) as source: + if source.format not in ("PNG", "JPEG"): + raise InvalidImage("invalid_format") + width, height = source.size + if width < 1 or height < 1 or width * height > MAX_PIXELS: + raise InvalidImage("invalid_dimensions") + source.verify() + + with Image.open(io.BytesIO(data)) as source: + oriented = ImageOps.exif_transpose(source) + try: + rgba = oriented.convert("RGBA") + try: + rgb = Image.new("RGB", rgba.size, "white") + alpha = rgba.getchannel("A") + try: + rgb.paste(rgba, mask=alpha) + finally: + alpha.close() + finally: + rgba.close() + finally: + oriented.close() + + try: + variants = [] + + size = fitted_size(rgb.width, rgb.height, 256) + with rgb.resize(size, Image.Resampling.LANCZOS) as thumb: + variants.append(encode_variant("thumbnail", thumb, "JPEG")) + + with ImageOps.fit( + rgb, + (128, 128), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) as crop: + variants.append(encode_variant("crop", crop, "JPEG")) + + size = fitted_size(rgb.width, rgb.height, 1280) + with rgb.resize(size, Image.Resampling.LANCZOS) as optimized: + variants.append(encode_variant("optimized", optimized, "WEBP")) + + return variants + finally: + rgb.close() + + except ( + UnidentifiedImageError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + OSError, + SyntaxError, + ValueError, + ) as exc: + if isinstance(exc, InvalidImage): + raise + raise InvalidImage("invalid_image") from None diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..106d0471 --- /dev/null +++ b/app/images.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import io +import unicodedata + +from PIL import Image, UnidentifiedImageError + +from app.errors import ApiError, invalid + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + +FORMAT_TO_MIME = { + "PNG": "image/png", + "JPEG": "image/jpeg", +} + + +def validate_filename(value: str | None) -> str: + if ( + value is None + or not 1 <= len(value) <= 255 + or not value.strip() + ): + raise invalid("Некорректное имя файла") + + for char in value: + if char in "/\\" or unicodedata.category(char) == "Cc": + raise invalid("Некорректное имя файла") + + return value + + +def inspect_image(data: bytes, declared_type: str) -> str: + if len(data) > MAX_BYTES: + raise ApiError( + 413, + "payload_too_large", + "Файл превышает 10 MiB", + ) + + if declared_type not in FORMAT_TO_MIME.values(): + raise ApiError( + 415, + "unsupported_media_type", + "Допустимы только image/png и image/jpeg", + ) + + try: + with Image.open(io.BytesIO(data)) as image: + detected = FORMAT_TO_MIME.get(image.format or "") + width, height = image.size + pixels = width * height + except Image.DecompressionBombError: + raise ApiError( + 413, + "payload_too_large", + "Слишком много пикселей", + ) from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError): + raise ApiError( + 415, + "unsupported_media_type", + "Файл не является корректным PNG/JPEG", + ) from None + + if detected is None or detected != declared_type: + raise ApiError( + 415, + "unsupported_media_type", + "Содержимое не соответствует Content-Type", + ) + + if width < 1 or height < 1 or pixels > MAX_PIXELS: + raise ApiError( + 413, + "payload_too_large", + "Слишком много пикселей", + ) + + try: + with Image.open(io.BytesIO(data)) as image: + image.verify() + + with Image.open(io.BytesIO(data)) as image: + image.load() + except (Image.DecompressionBombError, OSError, SyntaxError, ValueError): + raise ApiError( + 415, + "unsupported_media_type", + "Файл повреждён", + ) from None + + return detected diff --git a/app/instance.py b/app/instance.py new file mode 100644 index 00000000..58000465 --- /dev/null +++ b/app/instance.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import socket +from uuid import uuid4 + + +INSTANCE_ID = f"{socket.gethostname()}:{uuid4().hex[:12]}" diff --git a/app/job_db.py b/app/job_db.py new file mode 100644 index 00000000..db2a6e4f --- /dev/null +++ b/app/job_db.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import os + +import psycopg +from psycopg.rows import dict_row + + +def connect_db(): + return psycopg.connect( + host=os.getenv("PGHOST", "postgres"), + port=int(os.getenv("PGPORT", "5432")), + dbname=os.getenv("PGDATABASE", "messenger"), + user=os.getenv("PGUSER", "messenger"), + password=os.environ["PGPASSWORD"], + connect_timeout=3, + autocommit=True, + row_factory=dict_row, + options=( + "-c statement_timeout=3000 " + "-c lock_timeout=2000 " + "-c idle_in_transaction_session_timeout=60000" + ), + ) diff --git a/app/job_runner.py b/app/job_runner.py new file mode 100644 index 00000000..9bc74471 --- /dev/null +++ b/app/job_runner.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import hashlib +import json +import os +import resource +import sys +from uuid import UUID + + +def execute(job_id: UUID, lease_token: UUID) -> str: + from app.image_pipeline import InvalidImage, render_variants + from app.faults import trip_fault + from app.job_db import connect_db + from app.storage import _get, _put, bucket_name, get_client + + with connect_db() as db: + row = db.execute( + """ + SELECT j.attachment_id, j.pipeline_version, + a.staging_key, a.object_key, + a.content_type, a.size_bytes, a.sha256 + FROM attachment_jobs AS j + JOIN attachments AS a ON a.id = j.attachment_id + WHERE j.id = %s + AND j.status = 'processing' + AND j.lease_token = %s + AND j.lease_until > clock_timestamp() + AND a.pipeline_version = j.pipeline_version + AND a.status = 'processing' + """, + (job_id, lease_token), + ).fetchone() + + if row is None: + return "stale" + + data = _get(row["staging_key"]) + if ( + len(data) != row["size_bytes"] + or hashlib.sha256(data).hexdigest() != row["sha256"] + ): + raise InvalidImage("original_integrity_error") + + variants = render_variants(data) + + _put(row["object_key"], data, row["content_type"]) + + for variant in variants: + variant["object_key"] = ( + f"variants/{row['attachment_id']}/" + f"v{row['pipeline_version']}/{variant['name']}" + ) + _put( + variant["object_key"], + variant["data"], + variant["content_type"], + ) + + trip_fault(job_id, "after_objects") + + with connect_db() as db: + with db.transaction(): + job = db.execute( + """ + SELECT id + FROM attachment_jobs + WHERE id = %s + AND status = 'processing' + AND lease_token = %s + AND lease_until > clock_timestamp() + FOR UPDATE + """, + (job_id, lease_token), + ).fetchone() + + if job is None: + return "stale" + + attachment = db.execute( + """ + SELECT id + FROM attachments + WHERE id = %s + AND pipeline_version = %s + AND status = 'processing' + FOR UPDATE + """, + (row["attachment_id"], row["pipeline_version"]), + ).fetchone() + + if attachment is None: + return "stale" + + for variant in variants: + db.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, + name, object_key, content_type, + size_bytes, width, height, sha256 + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) + DO UPDATE SET + object_key = EXCLUDED.object_key, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256 + """, + ( + row["attachment_id"], row["pipeline_version"], + variant["name"], variant["object_key"], + variant["content_type"], variant["size_bytes"], + variant["width"], variant["height"], + variant["sha256"], + ), + ) + + db.execute( + """ + UPDATE attachments + SET status = 'ready', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = %s + """, + (row["attachment_id"],), + ) + + db.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', + lease_token = NULL, + lease_until = NULL, + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = %s + """, + (job_id,), + ) + + # Результат уже зафиксирован. Ошибка очистки не отменяет ready. + try: + get_client().delete_object( + Bucket=bucket_name(), + Key=row["staging_key"], + ) + with connect_db() as db: + db.execute( + """ + UPDATE attachments + SET staging_key = NULL + WHERE id = %s + AND staging_key = %s + AND status = 'ready' + """, + (row["attachment_id"], row["staging_key"]), + ) + except Exception: + pass + + return "ready" + + +def main(): + memory_bytes = int(os.getenv("JOB_MEMORY_MB", "512")) * 1024 * 1024 + cpu_seconds = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes)) + resource.setrlimit( + resource.RLIMIT_CPU, + (cpu_seconds, cpu_seconds + 1), + ) + + from app.image_pipeline import InvalidImage + + try: + outcome = execute(UUID(sys.argv[1]), UUID(sys.argv[2])) + result = {"outcome": outcome} + except InvalidImage: + result = {"outcome": "failed", "error_code": "invalid_image"} + except MemoryError: + result = {"outcome": "failed", "error_code": "resource_limit"} + except Exception: + result = {"outcome": "retry", "error_code": "processing_failed"} + + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 00000000..d570b38c --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import hashlib +import os +from uuid import UUID, uuid4 + +from app.db import pool +from app.object_locks import object_lock_id +from app.storage import bucket_name, put_object + + +PIPELINE_VERSION = 1 +JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) + +if not 1 <= JOB_MAX_ATTEMPTS <= 100: + raise ValueError("JOB_MAX_ATTEMPTS must be between 1 and 100") + + +async def accept_upload( + *, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + data: bytes, +) -> dict: + attachment_id = uuid4() + job_id = uuid4() + staging_key = f"staging/{attachment_id}/v{PIPELINE_VERSION}" + original_key = f"originals/{attachment_id}" + digest = hashlib.sha256(data).hexdigest() + + async with pool.connection() as conn: + await conn.execute( + "SET LOCAL idle_in_transaction_session_timeout = '60s'" + ) + await conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (object_lock_id(bucket_name(), staging_key),), + ) + + await put_object(staging_key, data, content_type) + + cursor = await conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, + content_type, size_bytes, sha256, + object_key, staging_key, pipeline_version, status + ) + VALUES ( + %s, %s, %s, %s, + %s, %s, %s, + %s, %s, %s, 'queued' + ) + RETURNING id, chat_id, owner_id, filename, + content_type, size_bytes, sha256, + status, error_code, created_at, updated_at + """, + ( + attachment_id, chat_id, owner_id, filename, + content_type, len(data), digest, + original_key, staging_key, PIPELINE_VERSION, + ), + ) + row = await cursor.fetchone() + + await conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, max_attempts + ) + VALUES (%s, %s, %s, %s) + """, + ( + job_id, attachment_id, + PIPELINE_VERSION, JOB_MAX_ATTEMPTS, + ), + ) + + await conn.execute( + """ + INSERT INTO attachment_outbox (id, job_id, attempt) + VALUES (%s, %s, 1) + """, + (uuid4(), job_id), + ) + + row["variants"] = [] + return row diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..8a72139c --- /dev/null +++ b/app/main.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from psycopg import InterfaceError, OperationalError +from psycopg.errors import LockNotAvailable, QueryCanceled +from psycopg_pool import PoolTimeout, TooManyRequests +from starlette.exceptions import HTTPException + +from app.cache import close_cache +from app.instance import INSTANCE_ID +from app.db import close_db, open_db +from app.errors import ApiError, error_response, invalid, not_found +from app.routers import attachments, chats, health, messages, sessions, users + + +logger = logging.getLogger("messenger.api") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + await open_db() + yield + finally: + await close_cache() + await close_db() + + +app = FastAPI( + title="Replicated PostgreSQL messenger — Lab 3", + docs_url=None, + redoc_url=None, + openapi_url=None, + lifespan=lifespan, +) + + +@app.middleware("http") +async def request_middleware(request: Request, call_next): + request.state.request_id = str(uuid4()) + + try: + response = await call_next(request) + + except ApiError as exc: + response = error_response(request, exc) + + except ( + OperationalError, + InterfaceError, + PoolTimeout, + TooManyRequests, + QueryCanceled, + LockNotAvailable, + ) as exc: + logger.warning( + "Database unavailable request_id=%s error_type=%s", + request.state.request_id, + type(exc).__name__, + ) + response = error_response( + request, + ApiError( + 503, + "unavailable", + "Сервис временно недоступен", + ), + ) + + except Exception as exc: + logger.error( + "Unhandled error request_id=%s error_type=%s", + request.state.request_id, + type(exc).__name__, + ) + response = error_response( + request, + ApiError( + 500, + "internal_error", + "Внутренняя ошибка сервера", + ), + ) + + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc): + return error_response(request, invalid()) + + +@app.exception_handler(HTTPException) +async def http_error_handler(request: Request, exc: HTTPException): + if exc.status_code == 404: + error = not_found() + else: + error = invalid("Недопустимый HTTP-запрос") + + return error_response(request, error) + + +app.include_router(health.router) +app.include_router(users.router) +app.include_router(sessions.router) +app.include_router(chats.router) +app.include_router(messages.router) + +app.include_router(attachments.router) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..4d1cb2f7 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import psycopg + + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +MIGRATION_LOCK_ID = 202602001 + + +def apply_migrations() -> None: + files = sorted(MIGRATIONS_DIR.glob("[0-9]*_*.sql")) + + if not files: + raise RuntimeError("Migration files not found") + + with psycopg.connect( + host=os.getenv("PGHOST", "postgres"), + port=int(os.getenv("PGPORT", "5432")), + dbname=os.getenv("PGDATABASE", "messenger"), + user=os.getenv("PGUSER", "messenger"), + password=os.environ["PGPASSWORD"], + connect_timeout=5, + options="-c statement_timeout=30000 -c lock_timeout=10000", + ) as conn: + conn.execute( + "SELECT pg_advisory_xact_lock(%s)", + (MIGRATION_LOCK_ID,), + ) + + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL + DEFAULT clock_timestamp() + ) + """ + ) + + for path in files: + version = path.name + content = path.read_bytes() + checksum = hashlib.sha256(content).hexdigest() + + row = conn.execute( + """ + SELECT checksum + FROM schema_migrations + WHERE version = %s + """, + (version,), + ).fetchone() + + if row is not None: + if row[0] != checksum: + raise RuntimeError( + f"Applied migration was modified: {version}" + ) + + print(f"Already applied: {version}", flush=True) + continue + + conn.execute(content.decode("utf-8"), prepare=False) + + conn.execute( + """ + INSERT INTO schema_migrations (version, checksum) + VALUES (%s, %s) + """, + (version, checksum), + ) + + print(f"Executed: {version} (pending commit)", flush=True) + + print("Migrations committed successfully", flush=True) + + +if __name__ == "__main__": + apply_migrations() \ No newline at end of file diff --git a/app/object_locks.py b/app/object_locks.py new file mode 100644 index 00000000..b232bdc3 --- /dev/null +++ b/app/object_locks.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import hashlib + + +def object_lock_id(bucket: str, object_key: str) -> int: + value = f"s3-object\0{bucket}\0{object_key}".encode("utf-8") + return int.from_bytes( + hashlib.sha256(value).digest()[:8], + byteorder="big", + signed=True, + ) diff --git a/app/pagination.py b/app/pagination.py new file mode 100644 index 00000000..839f7069 --- /dev/null +++ b/app/pagination.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import base64 +import binascii +import json +import re +from datetime import datetime +from uuid import UUID + +from fastapi import Request +from fastapi.responses import JSONResponse + +from app.errors import invalid +from app.responses import json_response, utc_text + + +def encode_cursor(scope: str, item: dict) -> str: + raw = json.dumps( + { + "scope": scope, + "created_at": utc_text(item["created_at"]), + "id": str(item["id"]), + }, + separators=(",", ":"), + ).encode("utf-8") + + return ( + base64.urlsafe_b64encode(raw) + .decode("ascii") + .rstrip("=") + ) + + +def decode_cursor( + cursor: str, + scope: str, +) -> tuple[datetime, UUID]: + if not cursor or len(cursor) > 2048: + raise invalid("Некорректный cursor") + + try: + padded = cursor + "=" * (-len(cursor) % 4) + + raw = base64.b64decode( + padded, + altchars=b"-_", + validate=True, + ) + + data = json.loads(raw.decode("utf-8")) + + if ( + not isinstance(data, dict) + or set(data) != {"scope", "created_at", "id"} + or data["scope"] != scope + ): + raise ValueError + + timestamp = data["created_at"] + + if ( + not isinstance(timestamp, str) + or not timestamp.endswith("Z") + or not isinstance(data["id"], str) + ): + raise ValueError + + created_at = datetime.fromisoformat( + timestamp[:-1] + "+00:00" + ) + + item_id = UUID(data["id"]) + + return created_at, item_id + + except ( + ValueError, + TypeError, + UnicodeError, + binascii.Error, + OverflowError, + ): + raise invalid("Некорректный cursor") from None + + +def pagination( + request: Request, + scope: str, +) -> tuple[int, tuple[datetime, UUID] | None]: + raw_limit = request.query_params.get("limit", "50") + + if ( + len(raw_limit) > 10 + or re.fullmatch(r"[0-9]+", raw_limit) is None + ): + raise invalid("Некорректный limit") + + limit = int(raw_limit) + + if not 1 <= limit <= 100: + raise invalid("Некорректный limit") + + raw_cursor = request.query_params.get("cursor") + after = None + + if raw_cursor is not None: + after = decode_cursor(raw_cursor, scope) + + return limit, after + + +def page_response( + rows: list[dict], + limit: int, + scope: str, +) -> JSONResponse: + items = rows[:limit] + next_cursor = None + + if len(rows) > limit: + next_cursor = encode_cursor(scope, items[-1]) + + return json_response( + { + "items": items, + "next_cursor": next_cursor, + } + ) diff --git a/app/publisher.py b/app/publisher.py new file mode 100644 index 00000000..e0262d2a --- /dev/null +++ b/app/publisher.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import logging +import time + +import pika + +from app.broker import QUEUE_NAME, connect_broker +from app.job_db import connect_db +from app.responses import utc_text + + +logger = logging.getLogger("messenger.publisher") + + +def publish_one(db, channel) -> bool: + with db.transaction(): + row = db.execute( + """ + SELECT o.id AS outbox_id, + o.job_id, + o.attempt, + j.attachment_id, + j.pipeline_version, + j.created_at + FROM attachment_outbox AS o + JOIN attachment_jobs AS j ON j.id = o.job_id + JOIN attachments AS a ON a.id = j.attachment_id + WHERE o.published_at IS NULL + AND o.available_at <= clock_timestamp() + AND j.available_at <= clock_timestamp() + AND j.status = 'queued' + AND j.attempt = o.attempt + AND a.status = 'queued' + AND a.pipeline_version = j.pipeline_version + ORDER BY o.available_at, o.id + LIMIT 1 + FOR UPDATE OF o SKIP LOCKED + """ + ).fetchone() + + if row is None: + return False + + payload = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": utc_text(row["created_at"]), + } + + channel.basic_publish( + exchange="", + routing_key=QUEUE_NAME, + body=json.dumps( + payload, + separators=(",", ":"), + ).encode("utf-8"), + properties=pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(row["outbox_id"]), + type="attachment.process.v1", + ), + mandatory=True, + ) + + db.execute( + """ + UPDATE attachment_outbox + SET published_at = clock_timestamp() + WHERE id = %s + """, + (row["outbox_id"],), + ) + + logger.info( + "published job_id=%s attempt=%s", + row["job_id"], + row["attempt"], + ) + return True + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + while True: + broker = None + try: + broker, channel = connect_broker() + channel.confirm_delivery() + + with connect_db() as db: + logger.info("publisher connected") + while True: + if not publish_one(db, channel): + broker.process_data_events(time_limit=0.5) + + except KeyboardInterrupt: + return + + except Exception as exc: + logger.warning( + "publisher reconnect error_type=%s", + type(exc).__name__, + ) + time.sleep(2) + + finally: + if broker is not None and broker.is_open: + try: + broker.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/app/replay.py b/app/replay.py new file mode 100644 index 00000000..6b1fb9d7 --- /dev/null +++ b/app/replay.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from uuid import UUID, uuid4 + +from app.job_db import connect_db +from app.responses import to_json +from app.storage import _get + + +def list_failed(): + with connect_db() as db: + return db.execute( + """ + SELECT id AS job_id, attachment_id, + pipeline_version, attempt, max_attempts, + error_code, updated_at + FROM attachment_jobs + WHERE status = 'failed' + ORDER BY updated_at, id + LIMIT 100 + """ + ).fetchall() + + +def replay(job_id, *, actor, reason, apply): + budget = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) + if not 1 <= budget <= 100: + raise ValueError("Invalid retry budget") + + with connect_db() as db: + with db.transaction(): + row = db.execute( + """ + SELECT j.*, + a.status AS attachment_status, + a.pipeline_version AS current_version, + a.staging_key, a.size_bytes, a.sha256 + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = %s + FOR UPDATE OF j, a + """, + (job_id,), + ).fetchone() + + if row is None: + raise ValueError("Unknown job") + + if ( + row["status"] != "failed" + or row["attachment_status"] != "failed" + or row["pipeline_version"] != row["current_version"] + ): + return { + "result": "noop", + "job_id": str(job_id), + "status": row["status"], + "reason": "job_is_not_current_failed_work", + } + + if not row["staging_key"]: + raise ValueError("Staging is unavailable") + + plan = { + "result": "replayed" if apply else "dry_run", + "job_id": str(job_id), + "previous_attempt": row["attempt"], + "new_attempt": row["attempt"] + 1, + "new_max_attempts": row["attempt"] + budget, + "previous_error_code": row["error_code"], + } + + if not apply: + return plan + + # Не ставим задачу в очередь, если исходник отсутствует + # или не совпадает с устойчиво принятыми метаданными. + db.execute( + "SET LOCAL idle_in_transaction_session_timeout = '60s'" + ) + original = _get(row["staging_key"]) + if ( + len(original) != row["size_bytes"] + or hashlib.sha256(original).hexdigest() != row["sha256"] + ): + raise ValueError("Staging integrity check failed") + + db.execute( + """ + INSERT INTO attachment_job_replays ( + id, job_id, actor, reason, + previous_attempt, new_attempt, previous_error_code + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + uuid4(), job_id, actor, reason, + row["attempt"], plan["new_attempt"], row["error_code"], + ), + ) + + db.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + attempt = %s, + max_attempts = %s, + error_code = NULL, + lease_token = NULL, + lease_until = NULL, + available_at = clock_timestamp(), + updated_at = clock_timestamp() + WHERE id = %s + """, + ( + plan["new_attempt"], + plan["new_max_attempts"], + job_id, + ), + ) + + db.execute( + """ + UPDATE attachments + SET status = 'queued', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = %s + """, + (row["attachment_id"],), + ) + + db.execute( + """ + INSERT INTO attachment_outbox (id, job_id, attempt) + VALUES (%s, %s, %s) + """, + (uuid4(), job_id, plan["new_attempt"]), + ) + + return plan + + +def main(): + parser = argparse.ArgumentParser() + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--list", action="store_true") + selection.add_argument("--job-id", type=UUID) + parser.add_argument("--actor") + parser.add_argument("--reason") + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + + if args.list and args.apply: + parser.error("--apply requires --job-id") + + if args.job_id: + if not args.actor or not 1 <= len(args.actor.strip()) <= 100: + parser.error("--actor must contain 1–100 characters") + if not args.reason or not 1 <= len(args.reason.strip()) <= 500: + parser.error("--reason must contain 1–500 characters") + + try: + if args.list: + result = list_failed() + else: + result = replay( + args.job_id, + actor=args.actor.strip(), + reason=args.reason.strip(), + apply=args.apply, + ) + except Exception as exc: + print(json.dumps({ + "result": "error", + "error_type": type(exc).__name__, + })) + raise SystemExit(1) + + print(json.dumps(to_json(result), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/app/responses.py b/app/responses.py new file mode 100644 index 00000000..4fed1a18 --- /dev/null +++ b/app/responses.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID + +from fastapi.responses import JSONResponse + + +def utc_text(value: datetime) -> str: + return ( + value.astimezone(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def to_json(value): + if isinstance(value, UUID): + return str(value) + + if isinstance(value, datetime): + return utc_text(value) + + if isinstance(value, dict): + return { + key: to_json(item) + for key, item in value.items() + } + + if isinstance(value, (list, tuple)): + return [to_json(item) for item in value] + + return value + + +def json_response(value, status: int = 200) -> JSONResponse: + return JSONResponse( + status_code=status, + content=to_json(value), + ) diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/attachments.py b/app/routers/attachments.py new file mode 100644 index 00000000..4d59eeac --- /dev/null +++ b/app/routers/attachments.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import asyncio +import hashlib +from contextlib import asynccontextmanager +from uuid import UUID + +from fastapi import APIRouter, Request +from fastapi.responses import Response +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from app.db import pool +from app.jobs import accept_upload +from app.dependencies import accessible_chat, current_user +from app.errors import ApiError, invalid, not_found +from app.images import MAX_BYTES, inspect_image, validate_filename +from app.responses import json_response +from app.storage import get_object +from app.validation import parse_uuid + + +router = APIRouter() +binary_slot = asyncio.Semaphore(1) + +PUBLIC_FIELDS = ( + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "status", + "created_at", + "updated_at", +) + + +@asynccontextmanager +async def binary_operation(): + try: + await asyncio.wait_for(binary_slot.acquire(), timeout=3) + except TimeoutError: + raise ApiError( + 503, + "unavailable", + "Обработка файлов временно занята", + ) from None + + try: + yield + finally: + binary_slot.release() + + +def public_attachment(row: dict) -> dict: + result = {field: row[field] for field in PUBLIC_FIELDS} + result["variants"] = row.get("variants", []) + result["error_code"] = row.get("error_code") + return result + + +async def read_image_body(request: Request) -> bytes: + lengths = request.headers.getlist("content-length") + + if len(lengths) > 1: + raise invalid("Некорректный Content-Length") + + expected = None + if lengths: + value = lengths[0] + if not value.isascii() or not value.isdecimal(): + raise invalid("Некорректный Content-Length") + + if len(value) > 20: + raise ApiError( + 413, "payload_too_large", "Файл превышает 10 MiB" + ) + + expected = int(value) + if expected > MAX_BYTES: + raise ApiError( + 413, "payload_too_large", "Файл превышает 10 MiB" + ) + + data = bytearray() + + async for chunk in request.stream(): + if len(data) + len(chunk) > MAX_BYTES: + raise ApiError( + 413, "payload_too_large", "Файл превышает 10 MiB" + ) + data.extend(chunk) + + if expected is not None and len(data) != expected: + raise invalid("Размер тела не соответствует Content-Length") + + return bytes(data) + + +async def accessible_attachment( + attachment_id: UUID, + user_id: UUID, +) -> dict: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, + a.filename, a.content_type, a.size_bytes, + a.sha256, a.object_key, a.status, + a.error_code, a.pipeline_version, + a.created_at, a.updated_at + FROM attachments AS a + WHERE a.id = %s + AND EXISTS ( + SELECT 1 + FROM chat_members AS cm + WHERE cm.chat_id = a.chat_id + AND cm.user_id = %s + ) + """, + (attachment_id, user_id), + ) + row = await cursor.fetchone() + + if row is not None: + cursor = await conn.execute( + """ + SELECT name, content_type, size_bytes, + width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = %s + AND pipeline_version = %s + ORDER BY name + """, + (row["id"], row["pipeline_version"]), + ) + variants = await cursor.fetchall() + row["_variant_objects"] = { + variant["name"]: variant + for variant in variants + } + row["variants"] = [ + { + key: value + for key, value in variant.items() + if key != "object_key" + } + for variant in variants + ] if row["status"] == "ready" else [] + + if row is None: + raise not_found() + + return row + + +@router.post("/api/v1/chats/{chat_id}/attachments") +async def upload_attachment(chat_id: str, request: Request): + user, _ = await current_user(request) + chat_uuid, _ = await accessible_chat(chat_id, user["id"]) + + filenames = request.query_params.getlist("filename") + if len(filenames) != 1: + raise invalid("Требуется одно имя файла") + + filename = validate_filename(filenames[0]) + + content_types = request.headers.getlist("content-type") + if len(content_types) != 1: + raise ApiError( + 415, + "unsupported_media_type", + "Требуется Content-Type изображения", + ) + + declared_type = content_types[0].split(";", 1)[0].strip().lower() + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError( + 415, + "unsupported_media_type", + "Допустимы только image/png и image/jpeg", + ) + + async with binary_operation(): + try: + data = await asyncio.wait_for( + read_image_body(request), + timeout=60, + ) + except ClientDisconnect: + raise invalid("Загрузка файла прервана") from None + except TimeoutError: + raise invalid("Превышено время загрузки файла") from None + + content_type = await run_in_threadpool( + inspect_image, + data, + declared_type, + ) + + row = await accept_upload( + chat_id=chat_uuid, + owner_id=user["id"], + filename=filename, + content_type=content_type, + data=data, + ) + + return json_response(public_attachment(row), 202) + + +@router.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + user, _ = await current_user(request) + + row = await accessible_attachment( + parse_uuid(attachment_id), + user["id"], + ) + + return json_response(public_attachment(row)) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + user, _ = await current_user(request) + + row = await accessible_attachment( + parse_uuid(attachment_id), + user["id"], + ) + + variants = request.query_params.getlist("variant") + if len(variants) > 1: + raise invalid("Допускается один вариант изображения") + + variant_name = variants[0] if variants else "original" + + if row["status"] != "ready": + raise ApiError( + 409, + "conflict", + "Вложение не готово к скачиванию", + ) + + if variant_name == "original": + selected = row + else: + selected = row["_variant_objects"].get(variant_name) + if selected is None: + raise not_found() + + async with binary_operation(): + data = await get_object(selected["object_key"]) + + if ( + len(data) != selected["size_bytes"] + or hashlib.sha256(data).hexdigest() != selected["sha256"] + ): + raise ApiError( + 503, + "unavailable", + "Изображение временно недоступно", + ) + + return Response( + content=data, + media_type=selected["content_type"], + headers={ + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + }, + ) diff --git a/app/routers/chats.py b/app/routers/chats.py new file mode 100644 index 00000000..36a27f25 --- /dev/null +++ b/app/routers/chats.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request + +from app.dependencies import accessible_chat, current_user +from app.errors import invalid +from app.pagination import page_response, pagination +from app.responses import json_response +from app.store import InvalidMembersError, store +from app.validation import parse_uuid, read_payload, valid_text + +router = APIRouter() + + +@router.post("/api/v1/chats") +async def create_chat(request: Request): + user, _ = await current_user(request) + + payload = await read_payload( + request, + {"title", "member_ids"}, + ) + + if not valid_text(payload["title"], 100): + raise invalid("Некорректный title") + + raw_members = payload["member_ids"] + + if ( + not isinstance(raw_members, list) + or not 1 <= len(raw_members) <= 100 + ): + raise invalid("Некорректный member_ids") + + member_ids = [parse_uuid(item) for item in raw_members] + + if len(set(member_ids)) != len(member_ids): + raise invalid("В member_ids есть дубликаты") + + if user["id"] not in member_ids: + raise invalid("Создатель должен входить в member_ids") + + try: + chat = await store.create_chat( + payload["title"], + member_ids, + ) + except InvalidMembersError: + raise invalid( + "В member_ids есть неизвестный пользователь" + ) from None + + return json_response(chat, 201) + + +@router.get("/api/v1/chats") +async def list_chats(request: Request): + user, _ = await current_user(request) + + scope = f"chats:{user['id']}" + limit, after = pagination(request, scope) + + rows = await store.list_chats(user["id"], limit, after) + + return page_response(rows, limit, scope) + + +@router.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + user, _ = await current_user(request) + + _, chat = await accessible_chat(chat_id, user["id"]) + + return json_response(chat) diff --git a/app/routers/health.py b/app/routers/health.py new file mode 100644 index 00000000..ed87a7c1 --- /dev/null +++ b/app/routers/health.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.db import check_db +from app.storage import check_storage + +router = APIRouter() + + +@router.get("/health/live") +async def health_live(): + return {"status": "ok"} + + +@router.get("/health/ready") +async def health_ready(): + await check_db() + await check_storage() + return {"status": "ok"} diff --git a/app/routers/messages.py b/app/routers/messages.py new file mode 100644 index 00000000..dea9a557 --- /dev/null +++ b/app/routers/messages.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import re + +from fastapi import APIRouter, Request + +from app.dependencies import accessible_chat, current_user +from app.errors import invalid +from app.idempotency import create_message_once +from app.pagination import page_response, pagination +from app.responses import json_response +from app.store import store +from app.validation import parse_uuid, read_payload + + +router = APIRouter() +KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}", re.ASCII) + + +@router.post("/api/v1/chats/{chat_id}/messages") +async def create_message(chat_id: str, request: Request): + user, _ = await current_user(request) + + chat_uuid, _ = await accessible_chat( + chat_id, + user["id"], + ) + + keys = request.headers.getlist("idempotency-key") + if len(keys) != 1 or KEY_RE.fullmatch(keys[0]) is None: + raise invalid("Требуется корректный Idempotency-Key") + + payload = await read_payload( + request, + {"text"}, + optional_fields={"attachment_ids"}, + ) + + text = payload["text"] + if not isinstance(text, str) or len(text) > 4000: + raise invalid("Некорректный текст сообщения") + + raw_attachment_ids = payload.get("attachment_ids", []) + if ( + not isinstance(raw_attachment_ids, list) + or len(raw_attachment_ids) > 10 + ): + raise invalid("Допускается список максимум из 10 вложений") + + attachment_ids = [ + parse_uuid(value) + for value in raw_attachment_ids + ] + + if len(set(attachment_ids)) != len(attachment_ids): + raise invalid("Вложения не должны повторяться") + + if not text.strip() and not attachment_ids: + raise invalid("Сообщение должно содержать текст или вложение") + + message = await create_message_once( + chat_uuid, + user["id"], + text, + keys[0], + attachment_ids=attachment_ids, + ) + + return json_response(message, 201) + + +@router.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + user, _ = await current_user(request) + + chat_uuid, _ = await accessible_chat( + chat_id, + user["id"], + ) + + scope = f"messages:{chat_uuid}" + limit, after = pagination(request, scope) + + rows = await store.list_messages( + chat_uuid, + limit, + after, + ) + + return page_response(rows, limit, scope) diff --git a/app/routers/sessions.py b/app/routers/sessions.py new file mode 100644 index 00000000..0dfaf2fb --- /dev/null +++ b/app/routers/sessions.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request +from fastapi.responses import Response + +from app.auth import ( + DUMMY_PASSWORD_HASH, + generate_token, + hash_token, + parse_basic_credentials, + verify_password, +) +from app.config import SESSION_TTL_SECONDS +from app.dependencies import current_user +from app.errors import invalid, unauthorized +from app.responses import json_response +from app.store import store + +router = APIRouter() + + +@router.post("/api/v1/auth/sessions") +async def create_session(request: Request): + credentials = parse_basic_credentials( + request.headers.get("authorization") + ) + + if credentials is None: + raise unauthorized("Basic") + + username, password = credentials + + user = await store.get_user_for_login(username) + + stored_hash = ( + DUMMY_PASSWORD_HASH + if user is None + else user["password_hash"] + ) + + password_ok = await verify_password(stored_hash, password) + + if user is None or not password_ok: + raise unauthorized("Basic") + + if await request.body(): + raise invalid("Тело запроса при входе не допускается") + + token = generate_token() + + session = await store.create_session( + user["id"], + hash_token(token), + SESSION_TTL_SECONDS, + ) + + public_user = { + field: user[field] + for field in ("id", "username", "display_name", "created_at") + } + + return json_response( + { + "token": token, + "token_type": "Bearer", + "expires_at": session["expires_at"], + "user": public_user, + }, + 201, + ) + + +@router.delete("/api/v1/auth/sessions/current") +async def revoke_session(request: Request): + _, token_digest = await current_user(request) + + revoked = await store.revoke_session(token_digest) + + if not revoked: + raise unauthorized() + + return Response(status_code=204) diff --git a/app/routers/users.py b/app/routers/users.py new file mode 100644 index 00000000..9adb1e2c --- /dev/null +++ b/app/routers/users.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request + +from app.auth import USERNAME_RE, hash_password +from app.dependencies import current_user +from app.errors import ApiError, invalid +from app.responses import json_response +from app.store import store +from app.validation import read_payload, valid_text + +router = APIRouter() + + +@router.post("/api/v1/users") +async def create_user(request: Request): + payload = await read_payload( + request, + {"username", "display_name", "password"}, + ) + + username = payload["username"] + display_name = payload["display_name"] + password = payload["password"] + + if ( + not isinstance(username, str) + or USERNAME_RE.fullmatch(username) is None + ): + raise invalid("Некорректный username") + + if not valid_text(display_name, 100): + raise invalid("Некорректное display_name") + + if not valid_text(password, 128) or len(password) < 12: + raise invalid("Пароль должен содержать от 12 до 128 символов") + + password_digest = await hash_password(password) + + user = await store.create_user( + username, + display_name, + password_digest, + ) + + if user is None: + raise ApiError( + 409, + "conflict", + "Username уже занят", + ) + + return json_response(user, 201) + + +@router.get("/api/v1/users/me") +async def get_current_user(request: Request): + user, _ = await current_user(request) + return json_response(user) diff --git a/app/s3_check.py b/app/s3_check.py new file mode 100644 index 00000000..4af81a19 --- /dev/null +++ b/app/s3_check.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import asyncio +import time + +from app.errors import ApiError +from app.storage import check_storage + + +async def main() -> None: + deadline = time.monotonic() + 90 + + while True: + try: + await check_storage() + except ApiError: + if time.monotonic() >= deadline: + raise SystemExit( + "S3 readiness failed: bucket unavailable " + "or API credentials rejected" + ) from None + + print("Waiting for S3 bucket...", flush=True) + await asyncio.sleep(2) + else: + print( + "S3 bucket is accessible with API credentials", + flush=True, + ) + return + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..25422610 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ( + BotoCoreError, + ClientError, +) + +from app.errors import ApiError + + +_client = None + + +def _unavailable() -> ApiError: + return ApiError( + 503, + "unavailable", + "Сервис временно недоступен", + ) + + + +def _credential(file_variable: str) -> str: + file_path = os.environ[file_variable] + value = Path(file_path).read_text().rstrip("\r\n") + if not value: + raise ValueError("Empty S3 credential file") + return value + +def bucket_name() -> str: + return os.environ["S3_BUCKET"] + + +def get_client(): + global _client + + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=_credential("S3_ACCESS_KEY_ID_FILE"), + aws_secret_access_key=_credential("S3_SECRET_ACCESS_KEY_FILE"), + region_name=os.getenv("S3_REGION", "us-east-1"), + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=10, + retries={ + "mode": "standard", + "total_max_attempts": 2, + }, + ), + ) + + return _client + + +def _put(key: str, data: bytes, content_type: str) -> None: + get_client().put_object( + Bucket=bucket_name(), + Key=key, + Body=data, + ContentType=content_type, + ) + + +def _get(key: str) -> bytes: + maximum = 10 * 1024 * 1024 + response = get_client().get_object( + Bucket=bucket_name(), + Key=key, + ) + body = response["Body"] + + try: + expected_size = response["ContentLength"] + if not 1 <= expected_size <= maximum: + raise _unavailable() + + data = body.read(maximum + 1) + + if len(data) != expected_size or len(data) > maximum: + raise _unavailable() + + return data + finally: + body.close() + + +async def put_object( + key: str, + data: bytes, + content_type: str, +) -> None: + try: + await asyncio.to_thread(_put, key, data, content_type) + except (BotoCoreError, ClientError, OSError, KeyError, ValueError): + raise _unavailable() from None + + +async def get_object(key: str) -> bytes: + try: + return await asyncio.to_thread(_get, key) + except (BotoCoreError, ClientError, OSError, KeyError, ValueError): + raise _unavailable() from None + + +def _check() -> None: + get_client().head_bucket(Bucket=bucket_name()) + + +async def check_storage() -> None: + try: + await asyncio.to_thread(_check) + except (BotoCoreError, ClientError, OSError, KeyError, ValueError): + raise _unavailable() from None diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..6f5e4abe --- /dev/null +++ b/app/store.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID, uuid4 + +from psycopg.errors import ForeignKeyViolation, UniqueViolation + +from app.db import pool + + +class InvalidMembersError(Exception): + pass + + +class PostgresStore: + async def create_user( + self, + username: str, + display_name: str, + password_hash: str, + ) -> dict | None: + try: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + INSERT INTO users ( + id, username, display_name, password_hash + ) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ) + user = await cursor.fetchone() + + return user + except UniqueViolation as exc: + if exc.diag.constraint_name == "users_username_unique": + return None + raise + + async def get_user(self, user_id: UUID) -> dict | None: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ) + return await cursor.fetchone() + + async def get_user_for_login(self, username: str) -> dict | None: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + SELECT id, username, display_name, + password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ) + return await cursor.fetchone() + + async def create_chat( + self, + title: str, + member_ids: list[UUID], + ) -> dict: + chat_id = uuid4() + + try: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + INSERT INTO chats (id, title) + VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, title), + ) + chat = await cursor.fetchone() + + async with conn.cursor() as cursor: + await cursor.executemany( + """ + INSERT INTO chat_members (chat_id, user_id) + VALUES (%s, %s) + """, + [(chat_id, user_id) for user_id in member_ids], + ) + + chat["member_ids"] = member_ids + + return chat + except ForeignKeyViolation as exc: + raise InvalidMembersError from exc + + async def get_accessible_chat( + self, + chat_id: UUID, + user_id: UUID, + ) -> dict | None: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm.user_id + FROM chat_members AS cm + WHERE cm.chat_id = c.id + ORDER BY cm.user_id + ) AS member_ids + FROM chats AS c + WHERE c.id = %s + AND EXISTS ( + SELECT 1 + FROM chat_members AS access + WHERE access.chat_id = c.id + AND access.user_id = %s + ) + """, + (chat_id, user_id), + ) + return await cursor.fetchone() + + async def list_chats( + self, + user_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[dict]: + params: list[object] = [user_id] + after_sql = "" + + if after is not None: + after_sql = "AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + + params.append(limit + 1) + + async with pool.connection() as conn: + cursor = await conn.execute( + f""" + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm.user_id + FROM chat_members AS cm + WHERE cm.chat_id = c.id + ORDER BY cm.user_id + ) AS member_ids + FROM chats AS c + JOIN chat_members AS access + ON access.chat_id = c.id + WHERE access.user_id = %s + {after_sql} + ORDER BY c.created_at, c.id + LIMIT %s + """, + params, + ) + return await cursor.fetchall() + + async def create_message( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + ) -> dict: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (uuid4(), chat_id, sender_id, text), + ) + message = await cursor.fetchone() + + return message + + async def list_messages( + self, + chat_id: UUID, + limit: int, + after: tuple[datetime, UUID] | None = None, + ) -> list[dict]: + params: list[object] = [chat_id] + after_sql = "" + + if after is not None: + after_sql = "AND (created_at, id) > (%s, %s)" + params.extend(after) + + params.append(limit + 1) + + async with pool.connection() as conn: + cursor = await conn.execute( + f""" + SELECT m.id, m.chat_id, m.sender_id, + m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id + FROM message_attachments AS ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) AS attachment_ids + FROM messages AS m + WHERE m.chat_id = %s + {after_sql} + ORDER BY created_at, id + LIMIT %s + """, + params, + ) + return await cursor.fetchall() + + async def create_session( + self, + user_id: UUID, + token_hash: str, + ttl_seconds: int, + ) -> dict: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + INSERT INTO sessions ( + token_hash, user_id, created_at, expires_at + ) + VALUES ( + %s, %s, statement_timestamp(), + statement_timestamp() + + (%s * INTERVAL '1 second') + ) + RETURNING expires_at + """, + (token_hash, user_id, ttl_seconds), + ) + session = await cursor.fetchone() + + return session + + async def get_session_user_id( + self, + token_hash: str, + ) -> UUID | None: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + SELECT user_id + FROM sessions + WHERE token_hash = %s + AND expires_at > statement_timestamp() + """, + (token_hash,), + ) + row = await cursor.fetchone() + + return None if row is None else row["user_id"] + + async def revoke_session(self, token_hash: str) -> bool: + async with pool.connection() as conn: + cursor = await conn.execute( + """ + DELETE FROM sessions + WHERE token_hash = %s + AND expires_at > statement_timestamp() + RETURNING token_hash + """, + (token_hash,), + ) + row = await cursor.fetchone() + + return row is not None + + +store = PostgresStore() \ No newline at end of file diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..38300e19 --- /dev/null +++ b/app/validation.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import Request + +from app.errors import ApiError, invalid + + +def parse_uuid(value: object) -> UUID: + if not isinstance(value, str): + raise invalid("Некорректный UUID") + + try: + return UUID(value) + except ValueError: + raise invalid("Некорректный UUID") from None + + +def valid_text(value: object, maximum: int) -> bool: + return ( + isinstance(value, str) + and 1 <= len(value) <= maximum + and bool(value.strip()) + ) + + +async def read_payload( + request: Request, + expected_fields: set[str], + optional_fields: set[str] | None = None, +) -> dict: + content_type = ( + request.headers.get("content-type", "") + .split(";", 1)[0] + .strip() + .lower() + ) + + if content_type != "application/json": + raise ApiError( + 415, + "unsupported_media_type", + "Требуется Content-Type: application/json", + ) + + try: + payload = await request.json() + except (ValueError, UnicodeError): + raise invalid("Некорректный JSON") from None + + allowed_fields = expected_fields | (optional_fields or set()) + + if ( + not isinstance(payload, dict) + or not expected_fields.issubset(payload) + or not set(payload).issubset(allowed_fields) + ): + raise invalid("Некорректные поля запроса") + + return payload diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..77cf168e --- /dev/null +++ b/app/worker.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import json +import logging +import os +import random +import subprocess +import sys +import time +from uuid import UUID, uuid4 + +from app.broker import QUEUE_NAME, connect_broker +from app.job_db import connect_db +from app.faults import trip_fault + + +logger = logging.getLogger("messenger.worker") + +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1")) +if WORKER_CONCURRENCY != 1: + raise ValueError("This worker supports WORKER_CONCURRENCY=1 only") + +JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120")) + +if JOB_TIMEOUT < 1 or LEASE_SECONDS < JOB_TIMEOUT + 15: + raise ValueError("Lease must exceed job timeout by at least 15 seconds") + + +def claim_job(payload: dict): + job_id = UUID(payload["job_id"]) + attachment_id = UUID(payload["attachment_id"]) + attempt = payload["attempt"] + version = payload["pipeline_version"] + + if ( + payload.get("schema_version") != 1 + or payload.get("type") != "attachment.process.v1" + or type(attempt) is not int + or attempt < 1 + or type(version) is not int + or version < 1 + ): + raise ValueError("Invalid job message") + + token = uuid4() + + with connect_db() as db: + with db.transaction(): + row = db.execute( + """ + SELECT j.id, j.status, j.attempt, + j.attachment_id, j.pipeline_version, + a.status AS attachment_status, + a.pipeline_version AS current_version + FROM attachment_jobs AS j + JOIN attachments AS a ON a.id = j.attachment_id + WHERE j.id = %s + FOR UPDATE OF j, a + """, + (job_id,), + ).fetchone() + + if ( + row is None + or row["attachment_id"] != attachment_id + or row["pipeline_version"] != version + or row["current_version"] != version + or row["attempt"] != attempt + or row["status"] != "queued" + or row["attachment_status"] != "queued" + ): + return None + + db.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', + lease_token = %s, + lease_until = clock_timestamp() + + make_interval(secs => %s), + updated_at = clock_timestamp() + WHERE id = %s + """, + (token, LEASE_SECONDS, job_id), + ) + + db.execute( + """ + UPDATE attachments + SET status = 'processing', + updated_at = clock_timestamp() + WHERE id = %s + """, + (attachment_id,), + ) + + return job_id, token + + +def finish_unsuccessful(job_id, token, *, permanent, error_code): + with connect_db() as db: + with db.transaction(): + row = db.execute( + """ + SELECT * + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job_id,), + ).fetchone() + + if ( + row is None + or row["status"] != "processing" + or row["lease_token"] != token + ): + return + + exhausted = row["attempt"] >= row["max_attempts"] + terminal = permanent or exhausted + status = "failed" if terminal else "queued" + code = ( + error_code if permanent else "retry_exhausted" + ) if terminal else None + + delay = random.uniform( + 0.5, + min(30.0, 2.0 ** min(row["attempt"], 5)), + ) + next_attempt = row["attempt"] if terminal else row["attempt"] + 1 + + db.execute( + """ + UPDATE attachment_jobs + SET status = %s, + attempt = %s, + error_code = %s, + lease_token = NULL, + lease_until = NULL, + available_at = clock_timestamp() + + make_interval(secs => %s), + updated_at = clock_timestamp() + WHERE id = %s + """, + (status, next_attempt, code, delay, job_id), + ) + + db.execute( + """ + UPDATE attachments + SET status = %s, + error_code = %s, + updated_at = clock_timestamp() + WHERE id = %s + AND pipeline_version = %s + AND status <> 'ready' + """, + ( + status, code, + row["attachment_id"], row["pipeline_version"], + ), + ) + + if not terminal: + db.execute( + """ + INSERT INTO attachment_outbox ( + id, job_id, attempt, available_at + ) + VALUES ( + %s, %s, %s, + clock_timestamp() + make_interval(secs => %s) + ) + """, + (uuid4(), job_id, next_attempt, delay), + ) + + logger.info( + "job transition job_id=%s status=%s reason=%s", + job_id, status, error_code, + ) + + +def recover_expired(): + with connect_db() as db: + rows = db.execute( + """ + SELECT id, lease_token + FROM attachment_jobs + WHERE status = 'processing' + AND lease_until < clock_timestamp() + ORDER BY lease_until + LIMIT 10 + """ + ).fetchall() + + for row in rows: + finish_unsuccessful( + row["id"], + row["lease_token"], + permanent=False, + error_code="lease_expired", + ) + + +def run_child(broker, job_id, token): + process = subprocess.Popen( + [ + sys.executable, + "-m", + "app.job_runner", + str(job_id), + str(token), + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + deadline = time.monotonic() + JOB_TIMEOUT + + try: + while process.poll() is None: + if time.monotonic() >= deadline: + process.kill() + process.communicate() + return {"outcome": "retry", "error_code": "processing_timeout"} + + broker.process_data_events(time_limit=0.2) + + output, _ = process.communicate() + + if process.returncode == 75: + logger.warning( + "child_fault job_id=%s stage=after_objects exit_code=75", + job_id, + ) + os._exit(75) + + if process.returncode != 0: + return {"outcome": "retry", "error_code": "processing_failed"} + + result = json.loads(output) + if result.get("outcome") not in ("ready", "stale", "failed", "retry"): + raise ValueError("Invalid child outcome") + return result + + finally: + if process.poll() is None: + process.kill() + process.communicate() + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + while True: + broker = None + try: + broker, channel = connect_broker() + channel.basic_qos(prefetch_count=1) + logger.info("worker connected") + + next_recovery = 0.0 + + while True: + if time.monotonic() >= next_recovery: + recover_expired() + next_recovery = time.monotonic() + 5 + + method, properties, body = channel.basic_get( + queue=QUEUE_NAME, + auto_ack=False, + ) + + if method is None: + broker.process_data_events(time_limit=0.5) + continue + + try: + if len(body) > 16_384: + raise ValueError("Oversized job message") + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError("Job must be an object") + claimed = claim_job(payload) + logger.info( + "delivery job_id=%s redelivered=%s", + payload["job_id"], + method.redelivered, + ) + except (ValueError, KeyError, TypeError): + logger.warning("invalid broker message rejected") + channel.basic_reject( + delivery_tag=method.delivery_tag, + requeue=False, + ) + continue + + if claimed is not None: + job_id, token = claimed + result = run_child(broker, job_id, token) + + if result["outcome"] in ("failed", "retry"): + finish_unsuccessful( + job_id, + token, + permanent=result["outcome"] == "failed", + error_code=result["error_code"], + ) + + if claimed is not None and result["outcome"] == "ready": + trip_fault(job_id, "before_ack") + + channel.basic_ack(delivery_tag=method.delivery_tag) + logger.info("ack job_id=%s", payload["job_id"]) + + except KeyboardInterrupt: + return + + except Exception as exc: + logger.warning( + "worker reconnect error_type=%s", + type(exc).__name__, + ) + time.sleep(2) + + finally: + if broker is not None and broker.is_open: + try: + broker.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/compose.queue.yaml b/compose.queue.yaml new file mode 100644 index 00000000..669fa7ea --- /dev/null +++ b/compose.queue.yaml @@ -0,0 +1,112 @@ +x-job-environment: &job-environment + PGHOST: postgres + PGPORT: "5432" + PGDATABASE: messenger + PGUSER: messenger + PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + BROKER_URL_FILE: /run/secrets/broker_url + +services: + broker: + image: rabbitmq:4.0.5-management + hostname: broker + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: + - CMD-SHELL + - rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_listener 5672 + interval: 20s + timeout: 30s + retries: 3 + start_period: 40s + cpus: "0.75" + mem_limit: 512m + stop_grace_period: 30s + restart: unless-stopped + + publisher: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.publisher"] + environment: *job-environment + secrets: + - broker_url + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + disable: true + cpus: "0.25" + mem_limit: 192m + stop_grace_period: 10s + restart: unless-stopped + + worker: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.worker"] + environment: + <<: *job-environment + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-60}" + JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-120}" + JOB_MEMORY_MB: "${JOB_MEMORY_MB:-512}" + WORKER_CONCURRENCY: "${WORKER_CONCURRENCY:-1}" + JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}" + secrets: + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-check: + condition: service_completed_successfully + healthcheck: + disable: true + cpus: "1.0" + mem_limit: 768m + stop_grace_period: 10s + restart: unless-stopped + + api: + environment: + JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}" + + # Старый cleanup пока запускается только явно: + # уборку staging и результатов новых попыток доработаем отдельно. + cleanup: + command: + - python + - -m + - app.cleanup_staging + - --apply + - --interval-seconds + - "3600" + environment: + STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-86400}" + +volumes: + broker_data: + +secrets: + broker_url: + file: ./secrets/broker_url diff --git a/compose.s3.yaml b/compose.s3.yaml new file mode 100644 index 00000000..8a67fc3e --- /dev/null +++ b/compose.s3.yaml @@ -0,0 +1,120 @@ +services: + s3: + image: chrislusf/seaweedfs:4.44 + entrypoint: ["weed"] + command: + - mini + - -dir=/data + - -ip=s3 + - -ip.bind=0.0.0.0 + - -admin.ui=false + - -bucket=messenger-attachments + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + - -s3.iam=false + - -s3.port.iceberg=0 + - -s3.port.lance=0 + volumes: + - s3_data:/data + secrets: + - s3_config + cpus: "1.0" + mem_limit: 768m + stop_grace_period: 20s + restart: unless-stopped + + s3-check: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.s3_check"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: + condition: service_started + healthcheck: + disable: true + cpus: "0.5" + mem_limit: 256m + restart: "no" + + migrate: + image: mtusi-messenger:lab5 + + api: + image: mtusi-messenger:lab5 + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-check: + condition: service_completed_successfully + + + cleanup: + build: + context: . + image: mtusi-messenger:lab5 + command: + - python + - -m + - app.cleanup_orphans + - --apply + - --grace-seconds + - "86400" + - --interval-seconds + - "3600" + environment: + PGHOST: postgres + PGPORT: "5432" + PGDATABASE: messenger + PGUSER: messenger + PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-check: + condition: service_completed_successfully + api: + condition: service_started + healthcheck: + disable: true + cpus: "0.25" + mem_limit: 256m + stop_grace_period: 30s + restart: unless-stopped + +volumes: + s3_data: + +secrets: + s3_config: + file: ./secrets/s3_config.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..76c84d86 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,132 @@ +name: mtusi2026lab5 + +x-db-environment: &db-environment + PGHOST: postgres + PGPORT: "5432" + PGDATABASE: messenger + PGUSER: messenger + PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: + - CMD-SHELL + - pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + interval: 2s + timeout: 2s + retries: 15 + start_period: 20s + cpus: "1.0" + mem_limit: 512m + restart: unless-stopped + + redis: + image: redis:7.2.5-bookworm + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --maxmemory + - 64mb + - --maxmemory-policy + - allkeys-lru + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 10 + start_period: 5s + cpus: "0.25" + mem_limit: 128m + restart: unless-stopped + + migrate: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.migrate"] + environment: *db-environment + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: "0.5" + mem_limit: 256m + restart: "no" + + api: + build: + context: . + image: mtusi-messenger:lab5 + command: + - python + - -m + - uvicorn + - app.main:app + - --host + - 0.0.0.0 + - --port + - "8080" + - --workers + - "1" + - --timeout-graceful-shutdown + - "5" + - --no-access-log + environment: + <<: *db-environment + REDIS_HOST: redis + REDIS_PORT: "6379" + USER_CACHE_TTL_SECONDS: "${USER_CACHE_TTL_SECONDS:-60}" + SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" + interval: 2s + timeout: 3s + retries: 3 + start_period: 10s + deploy: + replicas: 2 + cpus: "0.75" + mem_limit: 512m + stop_grace_period: 10s + restart: unless-stopped + + lb: + image: haproxy:3.0.5-bookworm + ports: + - "127.0.0.1:18085:8080" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_started + cpus: "0.25" + mem_limit: 128m + stop_grace_period: 10s + restart: unless-stopped + +volumes: + postgres_data: + redis_data: diff --git a/evidence/after_objects.json b/evidence/after_objects.json new file mode 100644 index 00000000..331a5b28 --- /dev/null +++ b/evidence/after_objects.json @@ -0,0 +1,58 @@ +{ + "scenario": "worker crash after S3 writes, before result COMMIT", + "attachment_id": "5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469", + "initial_job": { + "job_id": "48d3eef7-6769-4bbd-902e-55e52b89f67a", + "status": "queued", + "attempt": 1, + "outbox_count": 1, + "published": false + }, + "after_crash": { + "job_id": "48d3eef7-6769-4bbd-902e-55e52b89f67a", + "status": "processing", + "attempt": 1, + "lease_remaining_seconds": 116.606905, + "fault_consumed": true, + "variant_rows": 0 + }, + "objects_before_recovery": [ + "originals/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/crop", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/optimized", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/thumbnail" + ], + "confirmed_intermediate_state": { + "job_id": "48d3eef7-6769-4bbd-902e-55e52b89f67a", + "status": "processing", + "attempt": 1, + "lease_remaining_seconds": 111.482557, + "fault_consumed": true, + "variant_rows": 0 + }, + "final_job": { + "job_id": "48d3eef7-6769-4bbd-902e-55e52b89f67a", + "status": "ready", + "attempt": 2, + "outbox_count": 2, + "published": true + }, + "variant_rows": 3, + "objects_after_recovery": [ + "originals/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/crop", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/optimized", + "variants/5f4ea8f0-0efb-4fa2-bd7a-d4c07fa3a469/v1/thumbnail" + ], + "worker_events": [ + "worker-1 | 2026-11-17 14:49:11,362 INFO delivery job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a redelivered=False", + "worker-1 | 2026-11-17 14:49:14,573 WARNING child_fault job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a stage=after_objects exit_code=75", + "worker-1 | 2026-11-17 14:49:18,242 INFO delivery job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a redelivered=True", + "worker-1 | 2026-11-17 14:49:18,244 INFO ack job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a", + "worker-1 | 2026-11-17 14:51:14,782 INFO job transition job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a status=queued reason=lease_expired", + "worker-1 | 2026-11-17 14:51:17,341 INFO delivery job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a redelivered=False", + "worker-1 | 2026-11-17 14:51:20,984 INFO ack job_id=48d3eef7-6769-4bbd-902e-55e52b89f67a" + ], + "elapsed_s": 135.979, + "result": "PASS" +} diff --git a/evidence/before_ack.json b/evidence/before_ack.json new file mode 100644 index 00000000..7a874bb5 --- /dev/null +++ b/evidence/before_ack.json @@ -0,0 +1,44 @@ +{ + "scenario": "worker crash after ready commit, before ack", + "attachment_id": "821b61c9-7251-491b-ad51-c29870dd43ab", + "job_id": "92915f04-938a-4253-823f-8c299eb87788", + "initial_job": { + "job_id": "92915f04-938a-4253-823f-8c299eb87788", + "status": "queued", + "attempt": 1, + "outbox_count": 1, + "published": false + }, + "worker_before_fault": { + "started_at": "2026-11-17T14:39:01.016850053Z", + "restart_count": 0 + }, + "delivery_events": [ + "worker-1 | 2026-11-17 14:39:02,540 INFO delivery job_id=92915f04-938a-4253-823f-8c299eb87788 redelivered=False", + "worker-1 | 2026-11-17 14:39:05,774 WARNING fault_injection job_id=92915f04-938a-4253-823f-8c299eb87788 stage=before_ack exit_code=75", + "worker-1 | 2026-11-17 14:39:08,149 INFO delivery job_id=92915f04-938a-4253-823f-8c299eb87788 redelivered=True", + "worker-1 | 2026-11-17 14:39:08,150 INFO ack job_id=92915f04-938a-4253-823f-8c299eb87788" + ], + "worker_after_fault": { + "started_at": "2026-11-17T14:39:06.865588951Z", + "restart_count": 1 + }, + "final_job": { + "job_id": "92915f04-938a-4253-823f-8c299eb87788", + "status": "ready", + "attempt": 1, + "outbox_count": 1, + "published": true + }, + "fault": { + "consumed": true + }, + "variant_rows": 3, + "final_object_keys": [ + "originals/821b61c9-7251-491b-ad51-c29870dd43ab", + "variants/821b61c9-7251-491b-ad51-c29870dd43ab/v1/crop", + "variants/821b61c9-7251-491b-ad51-c29870dd43ab/v1/optimized", + "variants/821b61c9-7251-491b-ad51-c29870dd43ab/v1/thumbnail" + ], + "result": "PASS" +} diff --git a/evidence/cleanup_tests.txt b/evidence/cleanup_tests.txt new file mode 100644 index 00000000..66f58210 --- /dev/null +++ b/evidence/cleanup_tests.txt @@ -0,0 +1,13 @@ +test_busy_upload_lock_prevents_deletion (__main__.CleanupStagingTests.test_busy_upload_lock_prevents_deletion) ... ok +test_expired_failed_dry_run_is_non_destructive (__main__.CleanupStagingTests.test_expired_failed_dry_run_is_non_destructive) ... ok +test_expired_failed_staging_is_deleted_and_reference_cleared (__main__.CleanupStagingTests.test_expired_failed_staging_is_deleted_and_reference_cleared) ... ok +test_old_orphan_is_deleted_without_attachment_update (__main__.CleanupStagingTests.test_old_orphan_is_deleted_without_attachment_update) ... ok +test_processing_is_preserved_even_when_old (__main__.CleanupStagingTests.test_processing_is_preserved_even_when_old) ... ok +test_queued_is_preserved_even_when_old (__main__.CleanupStagingTests.test_queued_is_preserved_even_when_old) ... ok +test_recent_failed_state_preserves_old_object (__main__.CleanupStagingTests.test_recent_failed_state_preserves_old_object) ... ok +test_recent_object_is_preserved (__main__.CleanupStagingTests.test_recent_object_is_preserved) ... ok + +---------------------------------------------------------------------- +Ran 8 tests in 0.054s + +OK diff --git a/evidence/failed_replay.json b/evidence/failed_replay.json new file mode 100644 index 00000000..a43b01ab --- /dev/null +++ b/evidence/failed_replay.json @@ -0,0 +1,48 @@ +{ + "scenario": "failed staging integrity and audited replay", + "attachment_id": "9bcd1157-a974-444f-bba6-ec4fc558d287", + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "failed_job": { + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "status": "failed", + "attempt": 1, + "outbox_count": 1, + "published": true + }, + "rejected_replay": { + "result": "error", + "error_type": "ValueError" + }, + "dry_run": { + "result": "dry_run", + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "previous_attempt": 1, + "new_attempt": 2, + "new_max_attempts": 6, + "previous_error_code": "invalid_image" + }, + "applied_replay": { + "result": "replayed", + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "previous_attempt": 1, + "new_attempt": 2, + "new_max_attempts": 6, + "previous_error_code": "invalid_image" + }, + "final_job": { + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "status": "ready", + "attempt": 2, + "outbox_count": 2, + "published": true + }, + "ready_replay": { + "result": "noop", + "job_id": "c48f14d0-586d-4c76-9b0e-a1b90e2e0cd8", + "status": "ready", + "reason": "job_is_not_current_failed_work" + }, + "audit_rows": 1, + "object_count": 4, + "result": "PASS" +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..46786d8e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 2, "first_recovery_seconds": 2.089} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover_containers.json b/evidence/failover_containers.json new file mode 100644 index 00000000..9f1a7e1b --- /dev/null +++ b/evidence/failover_containers.json @@ -0,0 +1,18 @@ +[ + { + "container_id": "bfa59f929e063a129517dd173a77bd12a619f6e5bac20f0215f9fdafb92c300b", + "status": "running", + "running": true, + "started_at": "2026-11-17T15:28:48.449382903Z", + "finished_at": "2026-11-17T15:28:43.450087124Z", + "exit_code": 0 + }, + { + "container_id": "ff57dc533c533c0f3121e8c3377c2aa697cbd8da304adbebf4dce43b0a0998b8", + "status": "exited", + "running": false, + "started_at": "2026-11-17T15:28:47.804844296Z", + "finished_at": "2026-11-17T15:30:39.004319199Z", + "exit_code": 0 + } +] diff --git a/evidence/image_pipeline_tests.txt b/evidence/image_pipeline_tests.txt new file mode 100644 index 00000000..68b7dbb0 --- /dev/null +++ b/evidence/image_pipeline_tests.txt @@ -0,0 +1,11 @@ +test_contract_dimensions_and_original (test_image_pipeline.ImagePipelineTests.test_contract_dimensions_and_original) ... ok +test_exif_orientation (test_image_pipeline.ImagePipelineTests.test_exif_orientation) ... ok +test_invalid_input (test_image_pipeline.ImagePipelineTests.test_invalid_input) ... ok +test_rounding_down (test_image_pipeline.ImagePipelineTests.test_rounding_down) ... ok +test_small_image_not_upscaled_except_crop (test_image_pipeline.ImagePipelineTests.test_small_image_not_upscaled_except_crop) ... ok +test_transparency_becomes_white (test_image_pipeline.ImagePipelineTests.test_transparency_becomes_white) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 0.100s + +OK diff --git a/evidence/outbox_gap.json b/evidence/outbox_gap.json new file mode 100644 index 00000000..0173d7b1 --- /dev/null +++ b/evidence/outbox_gap.json @@ -0,0 +1,34 @@ +{ + "scenario": "API crash after COMMIT while publisher is stopped", + "attachment_id": "b5d15121-0129-4bf5-a249-2000aa16e911", + "before_crash": { + "job_id": "0010fe3a-2555-4ddf-a8da-9ce29abd9dd5", + "status": "queued", + "attempt": 1, + "outbox_count": 1, + "published": false + }, + "after_api_kill": { + "job_id": "0010fe3a-2555-4ddf-a8da-9ce29abd9dd5", + "status": "queued", + "attempt": 1, + "outbox_count": 1, + "published": false + }, + "after_publisher_start": { + "job_id": "0010fe3a-2555-4ddf-a8da-9ce29abd9dd5", + "status": "queued", + "attempt": 1, + "outbox_count": 1, + "published": true + }, + "final": { + "job_id": "0010fe3a-2555-4ddf-a8da-9ce29abd9dd5", + "status": "ready", + "attempt": 1, + "outbox_count": 1, + "published": true + }, + "ready_after_worker_start_s": 4.038, + "result": "PASS" +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..0ec6696b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/queued_restart.json b/evidence/queued_restart.json new file mode 100644 index 00000000..28809c83 --- /dev/null +++ b/evidence/queued_restart.json @@ -0,0 +1,38 @@ +{ + "scenario": "queued upload survives API and broker restart", + "attachment_id": "6876fe3c-1544-4e70-8011-b82c8b446ac9", + "message_id": "244c0f7c-c17b-4c17-867c-47e5da0e799a", + "job_before_restart": { + "status": "queued", + "attempt": 1, + "published": true + }, + "queue_before_restart": { + "name": "attachment.process.v1", + "durable": true, + "messages_ready": 1 + }, + "container_starts_before": { + "ea6e598f8967caf02b45629157b7156d60952001b4c73647212bb0124e0094f2": "2026-11-17T13:52:02.655967981Z", + "cd7a5131f262d163ca99084295bf4c291e971fb087af1dbd2020e13dd8e4d91f": "2026-11-17T13:52:02.172273085Z", + "d8f9d07940fea84f297d8562c8a4721cc5eac6b9bd684b0ea1edf9f51ffed1fd": "2026-11-17T13:56:52.523576495Z" + }, + "container_starts_after": { + "ea6e598f8967caf02b45629157b7156d60952001b4c73647212bb0124e0094f2": "2026-11-17T13:57:33.504017667Z", + "cd7a5131f262d163ca99084295bf4c291e971fb087af1dbd2020e13dd8e4d91f": "2026-11-17T13:57:32.869870682Z", + "d8f9d07940fea84f297d8562c8a4721cc5eac6b9bd684b0ea1edf9f51ffed1fd": "2026-11-17T13:57:34.831360909Z" + }, + "queue_after_restart": { + "name": "attachment.process.v1", + "durable": true, + "messages_ready": 1 + }, + "ready_after_worker_start_s": 5.857, + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "variants": [ + "crop", + "optimized", + "thumbnail" + ], + "result": "PASS" +} diff --git a/evidence/restart_containers.json b/evidence/restart_containers.json new file mode 100644 index 00000000..ef08afe8 --- /dev/null +++ b/evidence/restart_containers.json @@ -0,0 +1,163 @@ +{ + "before": [ + { + "service": "api", + "container_id": "bfa59f929e063a129517dd173a77bd12a619f6e5bac20f0215f9fdafb92c300b", + "started_at": "2026-11-17T15:18:11.387840892Z", + "status": "running", + "volumes": [] + }, + { + "service": "api", + "container_id": "ff57dc533c533c0f3121e8c3377c2aa697cbd8da304adbebf4dce43b0a0998b8", + "started_at": "2026-11-17T15:18:10.988865407Z", + "status": "running", + "volumes": [] + }, + { + "service": "broker", + "container_id": "26183862d5be1a9677d3981cdd4aa7ab41583720ea6ff51664b4b6ca1a30df54", + "started_at": "2026-11-17T14:37:29.569999892Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_broker_data" + ] + }, + { + "service": "cleanup", + "container_id": "d29213849d9d711eb711621a965f2f6720c6c4351b10705f06a07969fb43bc2d", + "started_at": "2026-11-17T15:22:37.758977743Z", + "status": "running", + "volumes": [] + }, + { + "service": "lb", + "container_id": "c7d31b9c6c1fb456d4b79b50521f00c6690e9fdf016fc53c9245f009296b242c", + "started_at": "2026-11-17T13:52:03.2138386Z", + "status": "running", + "volumes": [] + }, + { + "service": "postgres", + "container_id": "ad1aa87a07ef37319bb04c6e1caa05c5d7e0455bab6a8156603cb93ae3ed1638", + "started_at": "2026-11-17T13:51:56.606984187Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_postgres_data" + ] + }, + { + "service": "publisher", + "container_id": "123cc8ab2c6227e49096b6b41e3d59fc2b69d8e091a63e44b65662d72d13857b", + "started_at": "2026-11-17T15:04:26.1288457Z", + "status": "running", + "volumes": [] + }, + { + "service": "redis", + "container_id": "bc013b9a224e5d3aae5a3afed9880af92db0921acee34e9187ea7467eb5cafd4", + "started_at": "2026-11-17T13:51:56.57427508Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_redis_data" + ] + }, + { + "service": "s3", + "container_id": "4731679b480bde3edf68aaac640f2108a864efd437541891986480a404aefbf6", + "started_at": "2026-11-17T13:51:56.614820161Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_s3_data" + ] + }, + { + "service": "worker", + "container_id": "168ec66f5667bb59a99acd51a13fb36fcabc8d726d935007b3d4a605a8f0e798", + "started_at": "2026-11-17T15:18:59.941835146Z", + "status": "running", + "volumes": [] + } + ], + "result": "HOOK_COMPLETED", + "after": [ + { + "service": "api", + "container_id": "bfa59f929e063a129517dd173a77bd12a619f6e5bac20f0215f9fdafb92c300b", + "started_at": "2026-11-17T15:28:48.449382903Z", + "status": "running", + "volumes": [] + }, + { + "service": "api", + "container_id": "ff57dc533c533c0f3121e8c3377c2aa697cbd8da304adbebf4dce43b0a0998b8", + "started_at": "2026-11-17T15:28:47.804844296Z", + "status": "running", + "volumes": [] + }, + { + "service": "broker", + "container_id": "26183862d5be1a9677d3981cdd4aa7ab41583720ea6ff51664b4b6ca1a30df54", + "started_at": "2026-11-17T15:28:45.809717435Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_broker_data" + ] + }, + { + "service": "cleanup", + "container_id": "fe25b5c8dff54d1283675dcb8a31136f727b83e1bedef3a073df68c9084bd67c", + "started_at": "2026-11-17T15:28:49.052588548Z", + "status": "running", + "volumes": [] + }, + { + "service": "lb", + "container_id": "c7d31b9c6c1fb456d4b79b50521f00c6690e9fdf016fc53c9245f009296b242c", + "started_at": "2026-11-17T15:28:49.152009495Z", + "status": "running", + "volumes": [] + }, + { + "service": "postgres", + "container_id": "ad1aa87a07ef37319bb04c6e1caa05c5d7e0455bab6a8156603cb93ae3ed1638", + "started_at": "2026-11-17T15:28:44.877894569Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_postgres_data" + ] + }, + { + "service": "publisher", + "container_id": "8b583aba3eadce6ff307738c148164901325c1b763a7bfcd7a8cad1fbbc0df21", + "started_at": "2026-11-17T15:28:47.928076896Z", + "status": "running", + "volumes": [] + }, + { + "service": "redis", + "container_id": "bc013b9a224e5d3aae5a3afed9880af92db0921acee34e9187ea7467eb5cafd4", + "started_at": "2026-11-17T15:28:44.875553344Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_redis_data" + ] + }, + { + "service": "s3", + "container_id": "4731679b480bde3edf68aaac640f2108a864efd437541891986480a404aefbf6", + "started_at": "2026-11-17T15:28:46.590899487Z", + "status": "running", + "volumes": [ + "mtusi2026lab5_s3_data" + ] + }, + { + "service": "worker", + "container_id": "168ec66f5667bb59a99acd51a13fb36fcabc8d726d935007b3d4a605a8f0e798", + "started_at": "2026-11-17T15:28:47.957685083Z", + "status": "running", + "volumes": [] + } + ] +} diff --git a/evidence/retry_exhaustion.json b/evidence/retry_exhaustion.json new file mode 100644 index 00000000..d2354fa5 --- /dev/null +++ b/evidence/retry_exhaustion.json @@ -0,0 +1,64 @@ +{ + "scenario": "temporary S3 read failure exhausts bounded retries", + "attachment_id": "aa0f40d2-b242-4f79-95d6-3dd758e5f094", + "initial_job": { + "job_id": "e01d1f5b-9e96-4d61-b7cc-84320920a6dc", + "status": "queued", + "attempt": 1, + "max_attempts": 5, + "error_code": null, + "lease_cleared": true, + "outbox": [ + { + "attempt": 1, + "published": false, + "delay_seconds": 0.0 + } + ] + }, + "observed_attempts": [ + 1, + 2, + 3, + 4, + 5 + ], + "final_job": { + "job_id": "e01d1f5b-9e96-4d61-b7cc-84320920a6dc", + "status": "failed", + "attempt": 5, + "max_attempts": 5, + "error_code": "retry_exhausted", + "lease_cleared": true, + "outbox": [ + { + "attempt": 1, + "published": true, + "delay_seconds": 0.0 + }, + { + "attempt": 2, + "published": true, + "delay_seconds": 1.238091 + }, + { + "attempt": 3, + "published": true, + "delay_seconds": 0.55709 + }, + { + "attempt": 4, + "published": true, + "delay_seconds": 3.659677 + }, + { + "attempt": 5, + "published": true, + "delay_seconds": 4.988969 + } + ] + }, + "terminal_observation_seconds": 5, + "elapsed_s": 26.369, + "result": "PASS" +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..4834e35b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/staging_cleanup.json b/evidence/staging_cleanup.json new file mode 100644 index 00000000..093fe384 --- /dev/null +++ b/evidence/staging_cleanup.json @@ -0,0 +1,62 @@ +{ + "scenario": "scoped staging cleanup safety", + "queued_dry_run": { + "mode": "dry-run", + "grace_seconds": 0, + "attachment_id": "b8f27860-a4b8-49e4-a2a7-34a151b9c5bb", + "scanned": 2, + "ignored": 0, + "fresh": 0, + "busy": 0, + "active": 1, + "gone": 0, + "candidates": 1, + "deleted": 0 + }, + "queued_apply": { + "mode": "apply", + "grace_seconds": 0, + "attachment_id": "b8f27860-a4b8-49e4-a2a7-34a151b9c5bb", + "scanned": 2, + "ignored": 0, + "fresh": 0, + "busy": 0, + "active": 1, + "gone": 0, + "candidates": 1, + "deleted": 1 + }, + "attachment_id": "b8f27860-a4b8-49e4-a2a7-34a151b9c5bb", + "ready_dry_run": { + "mode": "dry-run", + "grace_seconds": 0, + "attachment_id": "b8f27860-a4b8-49e4-a2a7-34a151b9c5bb", + "scanned": 1, + "ignored": 0, + "fresh": 0, + "busy": 0, + "active": 0, + "gone": 0, + "candidates": 1, + "deleted": 0 + }, + "ready_apply": { + "mode": "apply", + "grace_seconds": 0, + "attachment_id": "b8f27860-a4b8-49e4-a2a7-34a151b9c5bb", + "scanned": 1, + "ignored": 0, + "fresh": 0, + "busy": 0, + "active": 0, + "gone": 0, + "candidates": 1, + "deleted": 1 + }, + "final_storage_state": { + "keys": [], + "status": "ready", + "staging_reference": false + }, + "result": "PASS" +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..812c5d73 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,42 @@ +global + log stdout format raw local0 + maxconn 512 + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 1s + timeout client 30s + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 3s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + hold other 1s + +frontend public_api + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1 + server-template api 1-4 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..586c0725 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,71 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length + CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT chats_title_length + CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx + ON chat_members (user_id, chat_id); + +CREATE INDEX chats_created_id_idx + ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT messages_sender_membership_fk + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + + CONSTRAINT messages_text_length + CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_chat_created_id_idx + ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT sessions_token_hash_format + CHECK (token_hash ~ '^[0-9a-f]{64}$'), + CONSTRAINT sessions_expiry_order + CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expires_at_idx + ON sessions (expires_at); + +CREATE INDEX sessions_user_id_idx + ON sessions (user_id); \ No newline at end of file diff --git a/migrations/002_message_idempotency.sql b/migrations/002_message_idempotency.sql new file mode 100644 index 00000000..3bd7b179 --- /dev/null +++ b/migrations/002_message_idempotency.sql @@ -0,0 +1,22 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + http_method TEXT NOT NULL DEFAULT 'POST', + idempotency_key TEXT NOT NULL, + request_text TEXT NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + PRIMARY KEY ( + user_id, chat_id, http_method, idempotency_key + ), + + CONSTRAINT message_idempotency_method + CHECK (http_method = 'POST'), + + CONSTRAINT message_idempotency_key_format + CHECK ( + char_length(idempotency_key) BETWEEN 1 AND 128 + AND idempotency_key ~ '^[A-Za-z0-9._:-]+$' + ) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a925a292 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,67 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CONSTRAINT attachments_filename_length + CHECK (char_length(filename) BETWEEN 1 AND 255), + + CONSTRAINT attachments_content_type + CHECK (content_type IN ('image/png', 'image/jpeg')), + + CONSTRAINT attachments_size_bytes + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT attachments_sha256_format + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + + CONSTRAINT attachments_status + CHECK (status = 'ready'), + + CONSTRAINT attachments_owner_membership_fk + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX attachments_chat_created_id_idx + ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + position SMALLINT NOT NULL, + + PRIMARY KEY (message_id, attachment_id), + + CONSTRAINT message_attachments_position_unique + UNIQUE (message_id, position), + + CONSTRAINT message_attachments_position_range + CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX message_attachments_attachment_idx + ON message_attachments (attachment_id); + +ALTER TABLE messages + DROP CONSTRAINT messages_text_length; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length + CHECK (char_length(text) BETWEEN 0 AND 4000); + +ALTER TABLE message_idempotency + ADD COLUMN request_attachment_ids UUID[] NOT NULL + DEFAULT ARRAY[]::UUID[]; + +ALTER TABLE message_idempotency + ADD CONSTRAINT message_idempotency_attachment_count + CHECK (cardinality(request_attachment_ids) <= 10); diff --git a/migrations/004_attachment_pipeline.sql b/migrations/004_attachment_pipeline.sql new file mode 100644 index 00000000..b3fc76ad --- /dev/null +++ b/migrations/004_attachment_pipeline.sql @@ -0,0 +1,101 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key TEXT, + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 + CHECK (pipeline_version >= 1), + ADD COLUMN error_code TEXT; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_error_state + CHECK ( + (status = 'failed' AND error_code IS NOT NULL) + OR + (status <> 'failed' AND error_code IS NULL) + ); + +CREATE UNIQUE INDEX attachments_staging_key_unique + ON attachments (staging_key) + WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + name TEXT NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL + CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + width INTEGER NOT NULL CHECK (width > 0), + height INTEGER NOT NULL CHECK (height > 0), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts INTEGER NOT NULL CHECK (max_attempts >= 1), + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + lease_token UUID, + lease_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + UNIQUE (attachment_id, pipeline_version), + + CHECK (attempt <= max_attempts), + + CHECK ( + ( + status = 'processing' + AND lease_token IS NOT NULL + AND lease_until IS NOT NULL + ) + OR + ( + status <> 'processing' + AND lease_token IS NULL + AND lease_until IS NULL + ) + ), + + CHECK ( + (status = 'failed' AND error_code IS NOT NULL) + OR + (status <> 'failed' AND error_code IS NULL) + ) +); + +CREATE INDEX attachment_jobs_recovery_idx + ON attachment_jobs (lease_until) + WHERE status = 'processing'; + +CREATE INDEX attachment_jobs_failed_idx + ON attachment_jobs (updated_at, id) + WHERE status = 'failed'; + +CREATE TABLE attachment_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + attempt INTEGER NOT NULL CHECK (attempt >= 1), + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + UNIQUE (job_id, attempt) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, id) + WHERE published_at IS NULL; diff --git a/migrations/005_fault_injections.sql b/migrations/005_fault_injections.sql new file mode 100644 index 00000000..7359f416 --- /dev/null +++ b/migrations/005_fault_injections.sql @@ -0,0 +1,8 @@ +CREATE TABLE pipeline_fault_injections ( + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + stage TEXT NOT NULL CHECK (stage IN ('before_ack')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + consumed_at TIMESTAMPTZ, + + PRIMARY KEY (job_id, stage) +); diff --git a/migrations/006_after_objects_fault.sql b/migrations/006_after_objects_fault.sql new file mode 100644 index 00000000..e56545cd --- /dev/null +++ b/migrations/006_after_objects_fault.sql @@ -0,0 +1,6 @@ +ALTER TABLE pipeline_fault_injections + DROP CONSTRAINT pipeline_fault_injections_stage_check; + +ALTER TABLE pipeline_fault_injections + ADD CONSTRAINT pipeline_fault_injections_stage_check + CHECK (stage IN ('before_ack', 'after_objects')); diff --git a/migrations/007_job_replays.sql b/migrations/007_job_replays.sql new file mode 100644 index 00000000..a3afe6f4 --- /dev/null +++ b/migrations/007_job_replays.sql @@ -0,0 +1,13 @@ +CREATE TABLE attachment_job_replays ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + actor TEXT NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100), + reason TEXT NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500), + previous_attempt INTEGER NOT NULL CHECK (previous_attempt >= 1), + new_attempt INTEGER NOT NULL, + previous_error_code TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + + CHECK (new_attempt = previous_attempt + 1), + UNIQUE (job_id, new_attempt) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..37cd00ab --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.43.124 +Pillow==13.0.0 +pika==1.3.2 diff --git a/scripts/dc b/scripts/dc new file mode 100755 index 00000000..833fad17 --- /dev/null +++ b/scripts/dc @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose \ + --project-name mtusi2026lab5 \ + --env-file .env \ + -f compose.yaml \ + -f compose.s3.yaml \ + -f compose.queue.yaml \ + "$@" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..1417fd51 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import time + + +ROOT = Path(__file__).resolve().parent.parent +DC = str(ROOT / "scripts/dc") + +STORES = ["postgres", "redis", "s3", "broker"] +PROCESSES = ["api", "lb", "publisher", "worker", "cleanup"] +SERVICES = STORES + PROCESSES + + +def compose(*args, timeout=60): + result = subprocess.run( + [DC, *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode: + raise RuntimeError("Compose failed: " + " ".join(args)) + return result.stdout.strip() + + +def snapshot(): + ids = compose("ps", "-a", "-q", *SERVICES).split() + if not ids: + raise RuntimeError("No lab5 containers found") + + result = subprocess.run( + ["docker", "inspect", *ids], + check=True, + capture_output=True, + text=True, + timeout=15, + ) + + records = [] + for item in json.loads(result.stdout): + records.append({ + "service": item["Config"]["Labels"]["com.docker.compose.service"], + "container_id": item["Id"], + "started_at": item["State"]["StartedAt"], + "status": item["State"]["Status"], + "volumes": sorted( + mount["Name"] + for mount in item["Mounts"] + if mount["Type"] == "volume" + ), + }) + return sorted(records, key=lambda item: (item["service"], item["container_id"])) + + +def main(): + report = {"before": snapshot(), "result": "INCOMPLETE"} + try: + print("Stopping API, LB and background processes...", flush=True) + compose("stop", "--timeout", "5", *PROCESSES) + + print("Restarting PostgreSQL, Redis, S3 and RabbitMQ...", flush=True) + compose("restart", "--no-deps", "--timeout", "10", *STORES) + + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + try: + compose( + "exec", "-T", "postgres", + "pg_isready", "-U", "messenger", "-d", "messenger", + timeout=4, + ) + break + except (RuntimeError, subprocess.TimeoutExpired): + time.sleep(1) + else: + raise RuntimeError("PostgreSQL readiness timeout") + + print("Starting API, LB and background processes...", flush=True) + compose("up", "-d", "--no-deps", "--no-build", *PROCESSES) + + after = snapshot() + report["after"] = after + + for service in SERVICES: + old = [item for item in report["before"] if item["service"] == service] + new = [item for item in after if item["service"] == service] + + if not old or len(old) != len(new): + raise RuntimeError(f"Unexpected container count: {service}") + + old_starts = {item["started_at"] for item in old} + if any(item["started_at"] in old_starts for item in new): + raise RuntimeError(f"Restart not observed: {service}") + + for service in STORES: + old_volumes = { + volume + for item in report["before"] if item["service"] == service + for volume in item["volumes"] + } + new_volumes = { + volume + for item in after if item["service"] == service + for volume in item["volumes"] + } + if not old_volumes or old_volumes != new_volumes: + raise RuntimeError(f"Persistent volumes changed: {service}") + + report["result"] = "HOOK_COMPLETED" + print("Restart hook completed; persistent volume names preserved.", flush=True) + + finally: + path = ROOT / "evidence" / "restart_containers.json" + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/setup_lab5.py b/scripts/setup_lab5.py new file mode 100644 index 00000000..87b045bf --- /dev/null +++ b/scripts/setup_lab5.py @@ -0,0 +1,67 @@ +from pathlib import Path +import json +import secrets + + +root = Path(__file__).resolve().parent.parent +directory = root / "secrets" + +targets = [ + root / ".env", + directory / "s3_config.json", + directory / "s3_access_key_id", + directory / "s3_secret_access_key", + directory / "broker_url", +] + +existing = [path for path in targets if path.exists()] +if existing: + raise SystemExit( + "STOP: local configuration already exists; " + "nothing changed. Do not delete or rotate it blindly." + ) + +directory.mkdir(mode=0o700, exist_ok=True) +directory.chmod(0o700) + +postgres_password = secrets.token_hex(24) +rabbit_password = secrets.token_hex(24) +access_key = "lab5" + secrets.token_hex(12) +secret_key = secrets.token_hex(32) + +config = { + "identities": [ + { + "name": "lab5", + "credentials": [ + { + "accessKey": access_key, + "secretKey": secret_key, + } + ], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + } + ] +} + +contents = { + root / ".env": ( + f"POSTGRES_PASSWORD={postgres_password}\n" + f"RABBITMQ_PASSWORD={rabbit_password}\n" + "SESSION_TTL_SECONDS=3600\n" + "USER_CACHE_TTL_SECONDS=60\n" + ), + directory / "s3_config.json": json.dumps(config, indent=2) + "\n", + directory / "s3_access_key_id": access_key + "\n", + directory / "s3_secret_access_key": secret_key + "\n", + directory / "broker_url": ( + f"amqp://messenger:{rabbit_password}@broker:5672/%2F\n" + ), +} + +for path, content in contents.items(): + with path.open("x") as stream: + path.chmod(0o600 if path.name == ".env" else 0o644) + stream.write(content) + +print("Created local lab5 configuration; secret values were not printed.") diff --git a/scripts/setup_s3_secrets.py b/scripts/setup_s3_secrets.py new file mode 100644 index 00000000..351e5001 --- /dev/null +++ b/scripts/setup_s3_secrets.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +from pathlib import Path +import secrets + + +ROOT = Path(__file__).resolve().parent.parent +DIRECTORY = ROOT / "secrets" +BUCKET = "messenger-attachments" + +KEY_FILES = ( + "s3_admin_access_key", + "s3_admin_secret_key", + "s3_access_key_id", + "s3_secret_access_key", +) + + +def configuration(values): + return { + "identities": [ + { + "name": "bootstrap-admin", + "credentials": [{ + "accessKey": values["s3_admin_access_key"], + "secretKey": values["s3_admin_secret_key"], + }], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "messenger-api", + "credentials": [{ + "accessKey": values["s3_access_key_id"], + "secretKey": values["s3_secret_access_key"], + }], + "actions": [ + f"Read:{BUCKET}", + f"Write:{BUCKET}", + f"List:{BUCKET}", + ], + }, + ] + } + + +def ensure_ignores(): + patterns = ( + ".env", + "/secrets/", + "*.bak", + "__pycache__/", + "*.py[cod]", + ) + for filename in (".gitignore", ".dockerignore"): + path = ROOT / filename + text = path.read_text() if path.exists() else "" + existing = set(text.splitlines()) + missing = [item for item in patterns if item not in existing] + if missing: + path.write_text( + text.rstrip("\n") + "\n" + "\n".join(missing) + "\n" + ) + + +def main(): + if DIRECTORY.is_symlink(): + raise SystemExit("Refusing a symlink at secrets/") + + paths = [DIRECTORY / name for name in KEY_FILES] + paths.append(DIRECTORY / "s3_config.json") + present = [path.exists() for path in paths] + + if any(present): + if not all(path.is_file() and not path.is_symlink() for path in paths): + raise SystemExit( + "Incomplete or unsafe secret files; nothing regenerated" + ) + + values = { + name: (DIRECTORY / name).read_text().rstrip("\r\n") + for name in KEY_FILES + } + if not all(values.values()): + raise SystemExit("Empty credential file; nothing regenerated") + + actual = json.loads( + (DIRECTORY / "s3_config.json").read_text() + ) + if actual != configuration(values): + raise SystemExit( + "Credentials and S3 configuration differ; nothing regenerated" + ) + message = "Existing credentials verified; keys unchanged." + else: + DIRECTORY.mkdir(mode=0o700, exist_ok=True) + DIRECTORY.chmod(0o700) + + values = { + "s3_admin_access_key": "admin-" + secrets.token_hex(12), + "s3_admin_secret_key": secrets.token_hex(32), + "s3_access_key_id": "api-" + secrets.token_hex(12), + "s3_secret_access_key": secrets.token_hex(32), + } + + for name, value in values.items(): + with (DIRECTORY / name).open("x") as stream: + stream.write(value + "\n") + + with (DIRECTORY / "s3_config.json").open("x") as stream: + json.dump(configuration(values), stream, indent=2) + stream.write("\n") + + message = "New credentials created; values are not printed." + + DIRECTORY.chmod(0o700) + for path in paths: + path.chmod(0o644) + + ensure_ignores() + print(message) + print("Secrets, .env, Python cache and .bak files are excluded.") + + +if __name__ == "__main__": + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..86312053 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +exec python3 - <<'PY' +import os +import subprocess + +target = os.environ.get("TARGET_INSTANCE_ID") +if not target: + raise SystemExit("TARGET_INSTANCE_ID is required") + +compose = ["./scripts/dc"] + +containers = subprocess.check_output( + compose + ["ps", "-q", "api"], + text=True, + timeout=10, +).split() + +if len(containers) < 2: + raise SystemExit("At least two running API containers are required") + +probe = ( + "import urllib.request; " + "r=urllib.request.urlopen(" + "'http://127.0.0.1:8080/health/ready', timeout=2); " + "print(r.headers['X-Instance-Id'])" +) + +matches = [] +for container in containers: + instance = subprocess.check_output( + ["docker", "exec", container, "python", "-c", probe], + text=True, + timeout=4, + ).strip() + + if instance == target: + matches.append(container) + +if len(matches) != 1: + raise SystemExit("Target instance was not resolved uniquely") + +container = matches[0] +print("Stopping target API container:", container[:12], flush=True) + +subprocess.run( + ["docker", "stop", "--time", "5", container], + check=True, + stdout=subprocess.DEVNULL, + timeout=15, +) + +running = subprocess.check_output( + ["docker", "inspect", "-f", "{{.State.Running}}", container], + text=True, + timeout=5, +).strip() + +if running != "false": + raise SystemExit("Target API container is still running") + +print("Target API container is stopped", flush=True) +PY diff --git a/student-tests/after_objects.py b/student-tests/after_objects.py new file mode 100644 index 00000000..e660b6bc --- /dev/null +++ b/student-tests/after_objects.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import secrets +import time +from uuid import UUID, uuid4 + +from queued_restart import ROOT, api, dc, http, wait_ready +from outbox_gap import snapshot +from before_ack import sql, object_keys + + +EVIDENCE = ROOT / "evidence" / "after_objects.json" + + +def intermediate_state(job_id): + return json.loads(sql(f""" + SELECT json_build_object( + 'job_id', j.id, + 'status', j.status, + 'attempt', j.attempt, + 'lease_remaining_seconds', + extract(epoch FROM (j.lease_until - clock_timestamp())), + 'fault_consumed', f.consumed_at IS NOT NULL, + 'variant_rows', ( + SELECT count(*) + FROM attachment_variants v + WHERE v.attachment_id = j.attachment_id + ) + ) + FROM attachment_jobs j + JOIN pipeline_fault_injections f ON f.job_id = j.id + WHERE j.id = '{job_id}'::uuid + AND f.stage = 'after_objects'; + """)) + + +def main(): + evidence = { + "scenario": "worker crash after S3 writes, before result COMMIT", + } + job_id = None + restore = False + + try: + wait_ready() + for service in ("publisher", "worker"): + if not dc("ps", "--status", "running", "-q", service): + raise AssertionError(f"Start {service} before this test") + + username = "obj_" + uuid4().hex[:16] + password = secrets.token_hex(16) + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "After objects test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + token = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + )["token"] + + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "After objects", + "member_ids": [user["id"]], + }, + ) + + restore = True + dc("stop", "worker") + if dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Worker is still running") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + + attachment_id = str(UUID(attachment["id"])) + path = f"/api/v1/attachments/{attachment_id}" + initial = snapshot(attachment_id) + job_id = str(UUID(initial["job_id"])) + + sql(f""" + INSERT INTO pipeline_fault_injections (job_id, stage) + VALUES ('{job_id}'::uuid, 'after_objects'); + """) + + evidence["attachment_id"] = attachment_id + evidence["initial_job"] = initial + print(f"Armed one-shot after_objects fault for job {job_id}") + + dc("start", "worker") + started = time.monotonic() + + deadline = started + 45 + while time.monotonic() < deadline: + middle = intermediate_state(job_id) + if middle["fault_consumed"]: + break + time.sleep(0.5) + else: + raise AssertionError("after_objects fault was not consumed") + + assert middle["status"] == "processing" + assert middle["attempt"] == 1 + assert middle["lease_remaining_seconds"] > 0 + assert middle["variant_rows"] == 0 + evidence["after_crash"] = middle + + current = api("GET", path, 200, token=token) + assert current["status"] == "processing" + assert current["variants"] == [] + api("GET", path + "/content", 409, token=token) + + expected_keys = sorted([ + f"originals/{attachment_id}", + f"variants/{attachment_id}/v1/thumbnail", + f"variants/{attachment_id}/v1/crop", + f"variants/{attachment_id}/v1/optimized", + ]) + + # Во время автоматического рестарта docker exec может быть недоступен. + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + try: + keys_before_recovery = object_keys(attachment_id) + break + except RuntimeError: + time.sleep(1) + else: + raise AssertionError("Could not inspect S3 after worker restart") + + assert keys_before_recovery == expected_keys + + # Подтверждаем, что осмотр S3 всё ещё был до восстановления lease. + confirmed_middle = intermediate_state(job_id) + assert confirmed_middle["status"] == "processing" + assert confirmed_middle["attempt"] == 1 + assert confirmed_middle["lease_remaining_seconds"] > 0 + assert confirmed_middle["variant_rows"] == 0 + + evidence["objects_before_recovery"] = keys_before_recovery + evidence["confirmed_intermediate_state"] = confirmed_middle + print( + "Fault consumed: processing, attempt=1, active lease, " + "S3 objects=4, variant rows=0, content=409" + ) + print("Waiting for automatic lease recovery...") + + deadline = started + 240 + while time.monotonic() < deadline: + current = api("GET", path, 200, token=token) + if current["status"] == "ready": + break + if current["status"] == "failed": + raise AssertionError( + "Attachment failed: " + str(current["error_code"]) + ) + time.sleep(1) + else: + raise AssertionError("Lease recovery did not reach ready") + + final = snapshot(attachment_id) + assert final["job_id"] == job_id + assert final["status"] == "ready" + assert final["attempt"] == 2 + assert final["outbox_count"] == 2 + assert current["error_code"] is None + + status, body = http("GET", path + "/content", token=token) + assert status == 200 and body == original + + variants = {item["name"]: item for item in current["variants"]} + assert len(current["variants"]) == 3 + assert set(variants) == {"thumbnail", "crop", "optimized"} + + for name, metadata in variants.items(): + status, body = http( + "GET", path + "/content?variant=" + name, + token=token, + ) + assert status == 200 + assert len(body) == metadata["size_bytes"] + assert hashlib.sha256(body).hexdigest() == metadata["sha256"] + + count = int(sql(f""" + SELECT count(*) + FROM attachment_variants + WHERE attachment_id = '{attachment_id}'::uuid; + """)) + assert count == 3 + + keys_after_recovery = object_keys(attachment_id) + assert keys_after_recovery == keys_before_recovery + + logs = dc("logs", "--no-color", "--tail=400", "worker") + events = [ + line for line in logs.splitlines() + if job_id in line + ] + assert any( + "child_fault" in line and "stage=after_objects" in line + for line in events + ) + assert any( + "status=queued" in line and "reason=lease_expired" in line + for line in events + ) + + evidence["final_job"] = final + evidence["variant_rows"] = count + evidence["objects_after_recovery"] = keys_after_recovery + evidence["worker_events"] = events + evidence["elapsed_s"] = round(time.monotonic() - started, 3) + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if job_id is not None: + sql(f""" + UPDATE pipeline_fault_injections + SET consumed_at = clock_timestamp() + WHERE job_id = '{job_id}'::uuid + AND stage = 'after_objects' + AND consumed_at IS NULL; + """) + finally: + try: + if restore: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print( + "PASS: crash after S3 writes recovered via expired lease; " + "same job, attempt=2, variants=3, objects=4" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/before_ack.py b/student-tests/before_ack.py new file mode 100644 index 00000000..0935e015 --- /dev/null +++ b/student-tests/before_ack.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import secrets +import subprocess +import time +from uuid import UUID, uuid4 + +from queued_restart import ROOT, api, dc, http, wait_ready +from outbox_gap import snapshot + + +EVIDENCE = ROOT / "evidence" / "before_ack.json" + + +def sql(query): + return dc( + "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", + "-c", query, + ) + + +def worker_state(): + ids = dc("ps", "-a", "-q", "worker").split() + if len(ids) != 1: + raise AssertionError("This test expects exactly one worker container") + + template = ( + '{"started_at":{{json .State.StartedAt}},' + '"restart_count":{{.RestartCount}}}' + ) + result = subprocess.run( + ["docker", "inspect", "--format", template, ids[0]], + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def object_keys(attachment_id): + code = """ +import json +import sys +from uuid import UUID +from app.storage import bucket_name, get_client + +attachment_id = str(UUID(sys.argv[1])) +client = get_client() +keys = [] +for prefix in ( + f"originals/{attachment_id}", + f"variants/{attachment_id}/", +): + for page in client.get_paginator("list_objects_v2").paginate( + Bucket=bucket_name(), Prefix=prefix, + ): + keys.extend(item["Key"] for item in page.get("Contents", [])) +print(json.dumps(sorted(keys))) +""" + return json.loads( + dc("exec", "-T", "worker", "python", "-c", code, attachment_id) + ) + + +def main(): + evidence = {"scenario": "worker crash after ready commit, before ack"} + job_id = None + restore = False + + try: + wait_ready() + if not dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Start worker before this test") + + username = "ack_" + uuid4().hex[:16] + password = secrets.token_hex(16) + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Before ack test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + token = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + )["token"] + + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Before ack", + "member_ids": [user["id"]], + }, + ) + + restore = True + dc("stop", "worker") + if dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Worker is still running") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = str(UUID(attachment["id"])) + path = f"/api/v1/attachments/{attachment_id}" + before = snapshot(attachment_id) + job_id = str(UUID(before["job_id"])) + + sql(f""" + INSERT INTO pipeline_fault_injections (job_id, stage) + VALUES ('{job_id}'::uuid, 'before_ack'); + """) + + evidence["attachment_id"] = attachment_id + evidence["job_id"] = job_id + evidence["initial_job"] = before + print(f"Armed one-shot before_ack fault for job {job_id}") + + dc("start", "worker") + started_state = worker_state() + evidence["worker_before_fault"] = started_state + + fault_line = f"fault_injection job_id={job_id} stage=before_ack" + redelivery_line = f"delivery job_id={job_id} redelivered=True" + ack_line = f"ack job_id={job_id}" + + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + logs = dc("logs", "--no-color", "--tail=300", "worker") + fault_at = logs.find(fault_line) + redelivery_at = logs.find(redelivery_line) + ack_at = logs.find(ack_line) + + if 0 <= fault_at < redelivery_at < ack_at: + break + time.sleep(1) + else: + raise AssertionError( + "Expected fault -> redelivered=True -> ack was not observed" + ) + + evidence["delivery_events"] = [ + line for line in logs.splitlines() + if job_id in line + ] + print("Observed: before_ack crash -> redelivered=True -> ack") + + after_state = worker_state() + assert ( + after_state["started_at"] != started_state["started_at"] + or after_state["restart_count"] > started_state["restart_count"] + ) + evidence["worker_after_fault"] = after_state + + consumed = json.loads(sql(f""" + SELECT json_build_object( + 'consumed', consumed_at IS NOT NULL + ) + FROM pipeline_fault_injections + WHERE job_id = '{job_id}'::uuid + AND stage = 'before_ack'; + """)) + assert consumed["consumed"] is True + + final = snapshot(attachment_id) + assert final["job_id"] == job_id + assert final["status"] == "ready" + assert final["attempt"] == 1 + assert final["outbox_count"] == 1 + + current = api("GET", path, 200, token=token) + assert current["status"] == "ready" + assert current["error_code"] is None + + variants = {item["name"]: item for item in current["variants"]} + assert len(current["variants"]) == 3 + assert set(variants) == {"thumbnail", "crop", "optimized"} + + status, body = http("GET", path + "/content", token=token) + assert status == 200 and body == original + + for name, metadata in variants.items(): + status, body = http( + "GET", path + "/content?variant=" + name, + token=token, + ) + assert status == 200 + assert len(body) == metadata["size_bytes"] + assert hashlib.sha256(body).hexdigest() == metadata["sha256"] + + variant_count = int(sql(f""" + SELECT count(*) + FROM attachment_variants + WHERE attachment_id = '{attachment_id}'::uuid; + """)) + assert variant_count == 3 + + keys = object_keys(attachment_id) + expected_keys = sorted([ + f"originals/{attachment_id}", + f"variants/{attachment_id}/v1/thumbnail", + f"variants/{attachment_id}/v1/crop", + f"variants/{attachment_id}/v1/optimized", + ]) + assert keys == expected_keys + + evidence["final_job"] = final + evidence["fault"] = consumed + evidence["variant_rows"] = variant_count + evidence["final_object_keys"] = keys + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if job_id is not None: + sql(f""" + UPDATE pipeline_fault_injections + SET consumed_at = clock_timestamp() + WHERE job_id = '{job_id}'::uuid + AND stage = 'before_ack' + AND consumed_at IS NULL; + """) + finally: + try: + if restore: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print( + "PASS: crash before ack caused broker redelivery; " + "ready result preserved, attempt=1, variants=3, objects=4" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/concurrent_idempotency.py b/student-tests/concurrent_idempotency.py new file mode 100644 index 00000000..90cf36ef --- /dev/null +++ b/student-tests/concurrent_idempotency.py @@ -0,0 +1,308 @@ +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, arguments, check, run +from resilience import history + + +WORKERS = 12 +ROUNDS = 3 + + +def main(): + args = arguments("Concurrent cross-replica idempotency") + + control = Client( + args.base_url, + args.ca_file, + timeout=10, + ) + alice, bob, eve, chat = control.bootstrap() + path = f"/api/v1/chats/{chat['id']}/messages" + + check( + history(control, alice, chat) == [], + "fresh test chat is not empty", + ) + + clients = [ + Client(args.base_url, args.ca_file, timeout=10) + for _ in range(WORKERS) + ] + + saved_messages = [] + round_results = [] + first_key = None + + for round_number in range(1, ROUNDS + 1): + key = uuid.uuid4().hex + text = f"Один ключ — одно сообщение: {round_number} {key}" + + if first_key is None: + first_key = key + + barrier = threading.Barrier(WORKERS) + + def send(index): + if index % 3 == 0: + body = {"text": text} + elif index % 3 == 1: + body = { + "attachment_ids": [], + "text": text, + } + else: + body = { + "text": text, + "attachment_ids": [], + } + + raw = json.dumps( + body, + ensure_ascii=False, + indent=2 if index % 2 else None, + ).encode("utf-8") + + barrier.wait(timeout=15) + + return clients[index].call( + "POST", + path, + expected=201, + actor=alice, + raw=raw, + headers={ + "Content-Type": "application/json", + "Idempotency-Key": key, + }, + schema="Message", + ) + + started = time.monotonic() + + with ThreadPoolExecutor(max_workers=WORKERS) as executor: + responses = list(executor.map(send, range(WORKERS))) + + elapsed = time.monotonic() - started + + original = responses[0]["json"] + + check( + all( + response["json"] == original + for response in responses + ), + "same key returned different Message objects", + ) + + instances = Counter( + response["headers"]["X-Instance-Id"] + for response in responses + ) + + check( + len(instances) >= 2, + "concurrent requests did not reach two replicas", + ) + + messages = history(control, bob, chat) + same_text = [ + message + for message in messages + if message["text"] == text + ] + + check( + same_text == [original], + "concurrent repeats created zero or multiple messages", + ) + check( + len(messages) == round_number, + "unexpected message count after concurrent round", + ) + + saved_messages.append(original) + + result = { + "round": round_number, + "requests": WORKERS, + "responses_201": len(responses), + "observed_instances": len(instances), + "responses_by_instance": dict(instances), + "unique_message_ids": len( + { + response["json"]["id"] + for response in responses + } + ), + "matching_history_messages": len(same_text), + "duration_seconds": round(elapsed, 3), + } + round_results.append(result) + print(json.dumps(result, ensure_ascii=False)) + + conflict = control.call( + "POST", + path, + expected=409, + actor=alice, + body={"text": "Другой текст с прежним ключом"}, + headers={"Idempotency-Key": first_key}, + ) + check( + conflict["json"]["error"]["code"] == "conflict", + "wrong error code for body conflict", + ) + + control.call( + "POST", + path, + expected=400, + actor=alice, + body={"text": "Без ключа"}, + ) + + for invalid_key in ("bad key", "x" * 129): + control.call( + "POST", + path, + expected=400, + actor=alice, + body={"text": "Некорректный ключ"}, + headers={"Idempotency-Key": invalid_key}, + ) + + control.call( + "POST", + path, + expected=404, + actor=eve, + body={"text": saved_messages[0]["text"]}, + headers={"Idempotency-Key": first_key}, + ) + + bob_message = control.message( + bob, + chat, + "Тот же ключ другого пользователя", + key=first_key, + )["json"] + + check( + bob_message["sender_id"] == bob["id"], + "wrong sender for another user's scope", + ) + check( + bob_message["id"] != saved_messages[0]["id"], + "key incorrectly shared between users", + ) + + second_chat = control.call( + "POST", + "/api/v1/chats", + expected=201, + actor=alice, + body={ + "title": "Другой чат для области ключа", + "member_ids": [alice["id"], bob["id"]], + }, + schema="Chat", + )["json"] + + other_chat_message = control.message( + alice, + second_chat, + "Тот же ключ в другом чате", + key=first_key, + )["json"] + + check( + other_chat_message["chat_id"] == second_chat["id"], + "wrong chat in scoped result", + ) + check( + other_chat_message["id"] != saved_messages[0]["id"], + "key incorrectly shared between chats", + ) + + replay = control.message( + alice, + chat, + saved_messages[0]["text"], + key=first_key, + )["json"] + + check( + replay == saved_messages[0], + "original result changed after scope tests", + ) + + messages = history(control, bob, chat) + expected = saved_messages + [bob_message] + + check( + len(messages) == len(expected), + "unexpected final message count", + ) + check( + {message["id"] for message in messages} + == {message["id"] for message in expected}, + "missing or extra messages in first chat", + ) + check( + history(control, bob, second_chat) == [other_chat_message], + "unexpected history in second chat", + ) + + control.call( + "DELETE", + "/api/v1/auth/sessions/current", + expected=204, + actor=alice, + ) + + control.call( + "POST", + path, + expected=401, + actor=alice, + body={"text": saved_messages[0]["text"]}, + headers={"Idempotency-Key": first_key}, + ) + + print( + json.dumps( + { + "rounds": ROUNDS, + "concurrent_requests_per_round": WORKERS, + "concurrent_requests_total": WORKERS * ROUNDS, + "body_conflict_409": True, + "missing_and_invalid_key_400": True, + "foreign_user_404": True, + "user_scope_independent": True, + "chat_scope_independent": True, + "replay_preserved": True, + "revoked_session_401": True, + "unexpected_messages": 0, + }, + ensure_ascii=False, + ) + ) + + print( + "PASS: concurrent cross-replica idempotency, " + "scope, validation and authorization checks." + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/failed_replay.py b/student-tests/failed_replay.py new file mode 100644 index 00000000..21494e7b --- /dev/null +++ b/student-tests/failed_replay.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import secrets +import subprocess +import time +from uuid import UUID, uuid4 + +from queued_restart import ROOT, DC, api, dc, http, wait_ready +from outbox_gap import snapshot +from before_ack import sql, object_keys + + +EVIDENCE = ROOT / "evidence" / "failed_replay.json" + + +def staging_write(attachment_id, *, original=None): + code = """ +import sys +from uuid import UUID +from app.job_db import connect_db +from app.storage import _put + +attachment_id = UUID(sys.argv[1]) +mode = sys.argv[2] + +with connect_db() as db: + row = db.execute( + ''' + SELECT staging_key, content_type + FROM attachments + WHERE id = %s + ''', + (attachment_id,), + ).fetchone() + +if row is None or not row["staging_key"]: + raise SystemExit(2) + +data = ( + b"deliberate synthetic staging corruption" + if mode == "corrupt" + else sys.stdin.buffer.read() +) +_put(row["staging_key"], data, row["content_type"]) +""" + mode = "corrupt" if original is None else "restore" + result = subprocess.run( + [ + DC, "exec", "-T", "api", + "python", "-c", code, attachment_id, mode, + ], + cwd=ROOT, + input=b"" if original is None else original, + capture_output=True, + timeout=90, + ) + if result.returncode: + raise RuntimeError(f"Staging operation failed: {mode}") + + +def replay_command(job_id, *, apply=False): + args = [ + DC, "exec", "-T", "worker", + "python", "-m", "app.replay", + "--job-id", job_id, + "--actor", "student-test", + "--reason", "Synthetic staging integrity fault repaired", + ] + if apply: + args.append("--apply") + + result = subprocess.run( + args, + cwd=ROOT, + capture_output=True, + text=True, + timeout=90, + ) + return result.returncode, json.loads(result.stdout) + + +def audit_count(job_id): + return int(sql(f""" + SELECT count(*) + FROM attachment_job_replays + WHERE job_id = '{job_id}'::uuid; + """)) + + +def wait_status(path, token, expected, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + current = api("GET", path, 200, token=token) + if current["status"] == expected: + return current + if expected == "ready" and current["status"] == "failed": + raise AssertionError( + "Replay failed: " + str(current["error_code"]) + ) + time.sleep(0.5) + raise AssertionError(f"Timeout waiting for {expected}") + + +def main(): + evidence = {"scenario": "failed staging integrity and audited replay"} + restore_worker = False + corrupted = False + attachment_id = None + original = None + + try: + assert sql( + "SELECT to_regclass('attachment_job_replays') IS NOT NULL;" + ) == "t", "Apply migration 007 first" + + wait_ready() + if not dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Start worker before this test") + + username = "rep_" + uuid4().hex[:16] + password = secrets.token_hex(16) + + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Failed replay test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + token = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + )["token"] + + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Failed replay", + "member_ids": [user["id"]], + }, + ) + + restore_worker = True + dc("stop", "worker") + if dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Worker is still running") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = str(UUID(attachment["id"])) + path = f"/api/v1/attachments/{attachment_id}" + job_id = str(UUID(snapshot(attachment_id)["job_id"])) + evidence["attachment_id"] = attachment_id + evidence["job_id"] = job_id + + # Отмечаем до вызова, чтобы finally попытался восстановить + # исходник даже при неопределённом результате S3-записи. + corrupted = True + staging_write(attachment_id) + dc("start", "worker") + + failed = wait_status(path, token, "failed") + assert failed["error_code"] == "invalid_image" + assert failed["variants"] == [] + + failed_job = snapshot(attachment_id) + assert failed_job["status"] == "failed" + assert failed_job["attempt"] == 1 + assert failed_job["outbox_count"] == 1 + assert audit_count(job_id) == 0 + + registry = json.loads( + dc("exec", "-T", "worker", "python", "-m", "app.replay", "--list") + ) + assert any(item["job_id"] == job_id for item in registry) + + api("GET", path + "/content", 409, token=token) + api( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 409, + token=token, + payload={"text": "", "attachment_ids": [attachment_id]}, + headers={"Idempotency-Key": str(uuid4())}, + ) + evidence["failed_job"] = failed_job + print("Corrupted staging produced failed/invalid_image; API returns 409") + + code, rejected = replay_command(job_id, apply=True) + assert code != 0 + assert rejected["result"] == "error" + assert snapshot(attachment_id) == failed_job + assert audit_count(job_id) == 0 + evidence["rejected_replay"] = rejected + print("Replay rejected corrupted staging without state changes or audit") + + staging_write(attachment_id, original=original) + corrupted = False + + code, dry_run = replay_command(job_id) + assert code == 0 and dry_run["result"] == "dry_run" + assert snapshot(attachment_id) == failed_job + assert audit_count(job_id) == 0 + evidence["dry_run"] = dry_run + print("Original restored; dry-run left job unchanged") + + code, applied = replay_command(job_id, apply=True) + assert code == 0 and applied["result"] == "replayed" + assert applied["new_attempt"] == 2 + + ready = wait_status(path, token, "ready") + final = snapshot(attachment_id) + assert final["job_id"] == job_id + assert final["status"] == "ready" + assert final["attempt"] == 2 + assert final["outbox_count"] == 2 + assert audit_count(job_id) == 1 + + status, body = http("GET", path + "/content", token=token) + assert status == 200 and body == original + + variants = {item["name"]: item for item in ready["variants"]} + assert len(ready["variants"]) == 3 + assert set(variants) == {"thumbnail", "crop", "optimized"} + + for name, metadata in variants.items(): + status, body = http( + "GET", path + "/content?variant=" + name, + token=token, + ) + assert status == 200 + assert len(body) == metadata["size_bytes"] + assert hashlib.sha256(body).hexdigest() == metadata["sha256"] + + keys_before_noop = object_keys(attachment_id) + assert len(keys_before_noop) == 4 + + code, noop = replay_command(job_id, apply=True) + assert code == 0 + assert noop["result"] == "noop" + assert noop["status"] == "ready" + assert snapshot(attachment_id) == final + assert audit_count(job_id) == 1 + assert object_keys(attachment_id) == keys_before_noop + + evidence["applied_replay"] = applied + evidence["final_job"] = final + evidence["ready_replay"] = noop + evidence["audit_rows"] = 1 + evidence["object_count"] = len(keys_before_noop) + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if corrupted and attachment_id is not None: + staging_write(attachment_id, original=original) + finally: + try: + if restore_worker: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print( + "PASS: failed registry, integrity guard, dry-run, audited replay " + "and ready noop verified; same job, attempt=2" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/outbox_gap.py b/student-tests/outbox_gap.py new file mode 100644 index 00000000..8dd95778 --- /dev/null +++ b/student-tests/outbox_gap.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import secrets +import time +from uuid import UUID, uuid4 + +from PIL import Image + +from queued_restart import ROOT, api, dc, http, wait_ready + + +EVIDENCE = ROOT / "evidence" / "outbox_gap.json" + + +def snapshot(attachment_id): + attachment_id = str(UUID(attachment_id)) + query = f""" + SELECT json_build_object( + 'job_id', j.id, + 'status', j.status, + 'attempt', j.attempt, + 'outbox_count', ( + SELECT count(*) + FROM attachment_outbox o + WHERE o.job_id = j.id + ), + 'published', EXISTS ( + SELECT 1 + FROM attachment_outbox o + WHERE o.job_id = j.id + AND o.published_at IS NOT NULL + ) + ) + FROM attachment_jobs j + WHERE j.attachment_id = '{attachment_id}'::uuid + AND j.pipeline_version = 1; + """ + result = dc( + "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", + "-c", query, + ) + if not result: + raise AssertionError("Job is missing") + return json.loads(result) + + +def main(): + evidence = { + "scenario": "API crash after COMMIT while publisher is stopped", + } + restore = False + + try: + wait_ready() + for service in ("publisher", "worker"): + if not dc("ps", "--status", "running", "-q", service): + raise AssertionError(f"Start {service} before this test") + + username = "gap_" + uuid4().hex[:16] + password = secrets.token_hex(16) + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Outbox gap test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + session = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + ) + token = session["token"] + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Outbox gap", + "member_ids": [user["id"]], + }, + ) + + restore = True + dc("stop", "publisher", "worker") + for service in ("publisher", "worker"): + if dc("ps", "--status", "running", "-q", service): + raise AssertionError(f"{service} is still running") + print("Publisher and worker stopped") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = attachment["id"] + path = f"/api/v1/attachments/{attachment_id}" + assert attachment["status"] == "queued" + + before = snapshot(attachment_id) + assert before["status"] == "queued" + assert before["attempt"] == 1 + assert before["outbox_count"] == 1 + assert before["published"] is False + + evidence["attachment_id"] = attachment_id + evidence["before_crash"] = before + print("Upload 202; job committed; outbox is not published") + + dc("kill", "--signal", "SIGKILL", "api") + + after_crash = snapshot(attachment_id) + assert after_crash == before + evidence["after_api_kill"] = after_crash + print("API killed with SIGKILL; committed job remains in PostgreSQL") + + dc("up", "-d", "--no-deps", "api") + wait_ready() + + # Прежняя сессия и то же вложение должны сохраниться. + current = api("GET", path, 200, token=token) + assert current["status"] == "queued" + api("GET", path + "/content", 409, token=token) + + dc("start", "publisher") + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + published = snapshot(attachment_id) + if published["published"]: + break + time.sleep(1) + else: + raise AssertionError("Pending outbox was not published") + + assert published["job_id"] == before["job_id"] + assert published["attempt"] == 1 + assert published["outbox_count"] == 1 + evidence["after_publisher_start"] = published + print("Publisher recovered the existing outbox without a new job") + + dc("start", "worker") + started = time.monotonic() + deadline = started + 90 + + while time.monotonic() < deadline: + current = api("GET", path, 200, token=token) + if current["status"] == "ready": + break + if current["status"] == "failed": + raise AssertionError( + "Processing failed: " + str(current["error_code"]) + ) + time.sleep(0.5) + else: + raise AssertionError("Ready timeout") + + status, body = http("GET", path + "/content", token=token) + assert status == 200 + assert body == original + + variants = {item["name"]: item for item in current["variants"]} + assert set(variants) == {"thumbnail", "crop", "optimized"} + + for name, expected_format in ( + ("thumbnail", "JPEG"), + ("crop", "JPEG"), + ("optimized", "WEBP"), + ): + status, body = http( + "GET", path + "/content?variant=" + name, + token=token, + ) + metadata = variants[name] + assert status == 200 + assert len(body) == metadata["size_bytes"] + assert hashlib.sha256(body).hexdigest() == metadata["sha256"] + with Image.open(io.BytesIO(body)) as image: + image.load() + assert image.format == expected_format + assert image.size == ( + metadata["width"], metadata["height"] + ) + + final = snapshot(attachment_id) + assert final["job_id"] == before["job_id"] + assert final["status"] == "ready" + assert final["attempt"] == 1 + assert final["outbox_count"] == 1 + + evidence["final"] = final + evidence["ready_after_worker_start_s"] = round( + time.monotonic() - started, 3 + ) + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if restore: + dc("up", "-d", "--no-deps", "api") + dc("start", "publisher", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print("PASS: committed outbox survived API crash and reached ready without a new job") + + +if __name__ == "__main__": + main() diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 00000000..018dad3e --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import concurrent.futures +import json +from pathlib import Path +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client + + +BASE_URL = "http://127.0.0.1:18082" +MAX_RESPONSE_SECONDS = 6.0 +WATCHDOG_SECONDS = 8.0 + +COMPOSE = [ + "docker", "compose", + "-f", "compose.yaml", + "-f", "compose.s3.yaml", + "--env-file", ".env", +] + + +def docker(*args): + return subprocess.check_output( + ["docker", *args], + cwd=ROOT, + text=True, + timeout=20, + stderr=subprocess.PIPE, + ).strip() + + +def main(): + pg_id = subprocess.check_output( + [*COMPOSE, "ps", "-q", "postgres"], + cwd=ROOT, + text=True, + timeout=20, + ).strip() + assert pg_id, "PostgreSQL container not found" + assert docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) == "false", "PostgreSQL is already paused" + + c = Client(BASE_URL, None) + alice, _, _, chat = c.bootstrap() + message = c.message(alice, chat, "postgres-pause regression")["json"] + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + + baseline_instances = set() + for _ in range(6): + response = c.call( + "GET", path, actor=alice, schema="MessagePage" + ) + assert any( + item["id"] == message["id"] + for item in response["json"]["items"] + ), "Baseline message is missing" + instance = response["headers"].get("X-Instance-Id") + assert instance, "Baseline response has no X-Instance-Id" + baseline_instances.add(instance) + + assert len(baseline_instances) >= 2, ( + "Baseline did not reach both API replicas" + ) + print("Baseline: history is available through both replicas", flush=True) + + def probe(): + started = time.monotonic() + try: + client = Client(BASE_URL, None) + response = client.call("GET", path, 503, actor=alice) + elapsed = time.monotonic() - started + instance = response["headers"].get("X-Instance-Id") + code = response["json"]["error"]["code"] + return { + "ok": bool(instance) and code == "unavailable", + "status": 503, + "code": code, + "instance": instance, + "elapsed_s": round(elapsed, 4), + } + except Exception as exc: + return { + "ok": False, + "error_type": type(exc).__name__, + "elapsed_s": round(time.monotonic() - started, 4), + } + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) + paused_by_test = False + try: + paused_by_test = True + docker("pause", pg_id) + assert docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) == "true" + print("PostgreSQL paused; requesting history", flush=True) + + futures = [executor.submit(probe) for _ in range(4)] + _, pending = concurrent.futures.wait( + futures, timeout=WATCHDOG_SECONDS + ) + assert not pending, "History requests exceeded the test watchdog" + + results = [future.result() for future in futures] + print(json.dumps(results, indent=2), flush=True) + + assert all(item["ok"] for item in results), ( + "Expected API JSON 503 unavailable with X-Instance-Id" + ) + assert all( + item["elapsed_s"] < MAX_RESPONSE_SECONDS + for item in results + ), "API response exceeded the declared deadline" + + failed_instances = {item["instance"] for item in results} + assert baseline_instances.issubset(failed_instances), ( + "Paused-DB requests did not exercise both baseline replicas" + ) + finally: + try: + if paused_by_test: + state = docker( + "inspect", "-f", "{{.State.Paused}}", pg_id + ) + if state == "true": + docker("unpause", pg_id) + print("PostgreSQL unpaused", flush=True) + finally: + executor.shutdown(wait=False, cancel_futures=True) + + recovery_started = time.monotonic() + recovery_deadline = recovery_started + 60 + while True: + try: + response = c.call( + "GET", path, actor=alice, schema="MessagePage" + ) + assert any( + item["id"] == message["id"] + for item in response["json"]["items"] + ), "Confirmed message disappeared" + break + except Exception: + if time.monotonic() >= recovery_deadline: + raise AssertionError("History did not recover") from None + time.sleep(1) + + report = { + "scenario": "postgres_pause_history", + "passed": True, + "max_response_seconds": MAX_RESPONSE_SECONDS, + "baseline_instances": sorted(baseline_instances), + "paused_requests": results, + "recovery_observed_after_s": round( + time.monotonic() - recovery_started, 4 + ), + "confirmed_message_preserved": True, + } + target = ROOT / "evidence" / "postgres_pause.json" + target.parent.mkdir(exist_ok=True) + target.write_text(json.dumps(report, indent=2) + "\n") + + print("PASS: bounded API 503 on both replicas; history recovered", flush=True) + + +if __name__ == "__main__": + main() diff --git a/student-tests/queued_restart.py b/student-tests/queued_restart.py new file mode 100644 index 00000000..3c48b7f1 --- /dev/null +++ b/student-tests/queued_restart.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +import base64 +import hashlib +import io +import json +from pathlib import Path +import secrets +import subprocess +import time +import urllib.error +import urllib.request +from uuid import uuid4 + +from PIL import Image + + +ROOT = Path(__file__).resolve().parent.parent +BASE = "http://127.0.0.1:18085" +DC = str(ROOT / "scripts/dc") +EVIDENCE = ROOT / "evidence" / "queued_restart.json" + + +def dc(*args, timeout=150): + result = subprocess.run( + [DC, *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode: + details = "\n".join( + part.strip() + for part in (result.stdout, result.stderr) + if part.strip() + ) + raise RuntimeError( + "Compose command failed: " + + " ".join(args) + + f" (exit={result.returncode})" + + "\n" + + details[-6000:] + ) + return result.stdout.strip() + + +def http(method, path, *, token=None, payload=None, data=None, headers=None): + request_headers = dict(headers or {}) + if token: + request_headers["Authorization"] = "Bearer " + token + if payload is not None: + data = json.dumps(payload).encode() + request_headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + BASE + path, + data=data, + headers=request_headers, + method=method, + ) + try: + response = urllib.request.urlopen(request, timeout=15) + except urllib.error.HTTPError as exc: + response = exc + + with response: + return response.status, response.read() + + +def api(method, path, expected, **kwargs): + status, body = http(method, path, **kwargs) + if status != expected: + raise AssertionError( + f"{method} {path}: expected {expected}, got {status}" + ) + return json.loads(body) if body else None + + +def wait_ready(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + if http("GET", "/health/ready")[0] == 200: + return + except (OSError, TimeoutError): + pass + time.sleep(1) + raise AssertionError("API readiness timeout") + + +def container_starts(): + result = {} + for service in ("api", "broker"): + ids = dc("ps", "-q", service).split() + if not ids: + raise AssertionError(f"No running containers for {service}") + for container_id in ids: + inspected = subprocess.run( + [ + "docker", "inspect", + "--format", "{{.State.StartedAt}}", + container_id, + ], + check=True, + capture_output=True, + text=True, + ) + result[container_id] = inspected.stdout.strip() + return result + + +def job_snapshot(attachment_id): + code = """ +import json +import sys +from uuid import UUID +from app.job_db import connect_db + +with connect_db() as db: + row = db.execute( + ''' + SELECT j.status, j.attempt, + EXISTS ( + SELECT 1 FROM attachment_outbox o + WHERE o.job_id = j.id + AND o.attempt = j.attempt + AND o.published_at IS NOT NULL + ) AS published + FROM attachment_jobs j + WHERE j.attachment_id = %s + AND j.pipeline_version = 1 + ''', + (UUID(sys.argv[1]),), + ).fetchone() +print(json.dumps(row)) +""" + return json.loads( + dc("exec", "-T", "publisher", "python", "-c", code, attachment_id) + ) + + +def queue_snapshot(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + output = dc( + "exec", "-T", "broker", + "rabbitmqctl", "-q", "list_queues", + "name", "durable", "messages_ready", + timeout=20, + ) + for line in output.splitlines(): + fields = line.split() + if len(fields) == 3 and fields[0] == "attachment.process.v1": + return { + "name": fields[0], + "durable": fields[1].lower() == "true", + "messages_ready": int(fields[2]), + } + except (RuntimeError, subprocess.TimeoutExpired): + pass + time.sleep(2) + raise AssertionError("Queue inspection timeout") + + +def main(): + evidence = {"scenario": "queued upload survives API and broker restart"} + stopped = False + + try: + wait_ready() + if not dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Start worker before running this test") + + username = "lab5_" + uuid4().hex[:16] + password = secrets.token_hex(16) + + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Queue restart test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + session = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + ) + token = session["token"] + + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Queued restart", + "member_ids": [user["id"]], + }, + ) + + dc("stop", "worker") + stopped = True + if dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Worker is still running") + print("Worker stopped") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = attachment["id"] + path = f"/api/v1/attachments/{attachment_id}" + assert attachment["status"] == "queued" + evidence["attachment_id"] = attachment_id + print(f"Upload accepted: {attachment_id}, status=queued") + + api("GET", path + "/content", 409, token=token) + + message = api( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=token, + payload={"text": "", "attachment_ids": [attachment_id]}, + headers={"Idempotency-Key": str(uuid4())}, + ) + evidence["message_id"] = message["id"] + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + snapshot = job_snapshot(attachment_id) + if snapshot and snapshot["published"]: + break + time.sleep(1) + else: + raise AssertionError("Outbox was not confirmed as published") + + evidence["job_before_restart"] = snapshot + before_queue = queue_snapshot() + assert before_queue["durable"] + assert before_queue["messages_ready"] >= 1 + evidence["queue_before_restart"] = before_queue + + before = container_starts() + dc("restart", "api", "broker") + wait_ready() + after_queue = queue_snapshot() + after = container_starts() + + assert set(before) == set(after) + assert all(before[key] != after[key] for key in before) + assert after_queue["durable"] + assert after_queue["messages_ready"] >= 1 + evidence["container_starts_before"] = before + evidence["container_starts_after"] = after + evidence["queue_after_restart"] = after_queue + print("API and broker restarted; queued message remains in broker") + + current = api("GET", path, 200, token=token) + assert current["status"] == "queued" + api("GET", path + "/content", 409, token=token) + + history = api( + "GET", + f"/api/v1/chats/{chat['id']}/messages?limit=100", + 200, + token=token, + ) + assert any(item["id"] == message["id"] for item in history["items"]) + + dc("start", "worker") + started = time.monotonic() + + deadline = started + 90 + while time.monotonic() < deadline: + current = api("GET", path, 200, token=token) + if current["status"] == "ready": + break + if current["status"] == "failed": + raise AssertionError( + "Attachment failed: " + str(current["error_code"]) + ) + time.sleep(0.5) + else: + raise AssertionError("Attachment did not become ready") + + status, downloaded = http("GET", path + "/content", token=token) + assert status == 200 and downloaded == original + + variants = {item["name"]: item for item in current["variants"]} + assert set(variants) == {"thumbnail", "crop", "optimized"} + + for name, fmt in ( + ("thumbnail", "JPEG"), + ("crop", "JPEG"), + ("optimized", "WEBP"), + ): + status, body = http( + "GET", path + "/content?variant=" + name, token=token, + ) + assert status == 200 + metadata = variants[name] + assert len(body) == metadata["size_bytes"] + assert hashlib.sha256(body).hexdigest() == metadata["sha256"] + with Image.open(io.BytesIO(body)) as image: + image.load() + assert image.format == fmt + assert image.size == (metadata["width"], metadata["height"]) + + evidence["ready_after_worker_start_s"] = round( + time.monotonic() - started, 3 + ) + evidence["original_sha256"] = hashlib.sha256(original).hexdigest() + evidence["variants"] = list(variants) + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if stopped: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print("PASS: queued task survived API/broker restart; original and variants verified") + + +if __name__ == "__main__": + main() diff --git a/student-tests/retry_exhaustion.py b/student-tests/retry_exhaustion.py new file mode 100644 index 00000000..22896e8e --- /dev/null +++ b/student-tests/retry_exhaustion.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import base64 +import json +import secrets +import time +from uuid import UUID, uuid4 + +from queued_restart import ROOT, api, dc, wait_ready +from before_ack import sql +from failed_replay import staging_write + + +EVIDENCE = ROOT / "evidence" / "retry_exhaustion.json" + + +def job_details(attachment_id): + return json.loads(sql(f""" + SELECT json_build_object( + 'job_id', j.id, + 'status', j.status, + 'attempt', j.attempt, + 'max_attempts', j.max_attempts, + 'error_code', j.error_code, + 'lease_cleared', + j.lease_token IS NULL AND j.lease_until IS NULL, + 'outbox', ( + SELECT json_agg( + json_build_object( + 'attempt', o.attempt, + 'published', o.published_at IS NOT NULL, + 'delay_seconds', + extract(epoch FROM (o.available_at - o.created_at)) + ) + ORDER BY o.attempt + ) + FROM attachment_outbox o + WHERE o.job_id = j.id + ) + ) + FROM attachment_jobs j + WHERE j.attachment_id = '{attachment_id}'::uuid + AND j.pipeline_version = 1; + """)) + + +def remove_staging(attachment_id): + code = """ +import sys +from uuid import UUID +from app.job_db import connect_db +from app.storage import bucket_name, get_client + +with connect_db() as db: + row = db.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (UUID(sys.argv[1]),), + ).fetchone() + +if row is None or not row["staging_key"]: + raise RuntimeError("Expected staging reference") + +get_client().delete_object( + Bucket=bucket_name(), + Key=row["staging_key"], +) +""" + dc("exec", "-T", "api", "python", "-c", code, attachment_id) + + +def main(): + evidence = {"scenario": "temporary S3 read failure exhausts bounded retries"} + restore_worker = False + restore_staging = False + attachment_id = None + original = None + + try: + wait_ready() + for service in ("publisher", "worker"): + if not dc("ps", "--status", "running", "-q", service): + raise AssertionError(f"Start {service} before this test") + + username = "rty_" + uuid4().hex[:16] + password = secrets.token_hex(16) + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Retry exhaustion test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + token = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + )["token"] + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Retry exhaustion", + "member_ids": [user["id"]], + }, + ) + + restore_worker = True + dc("stop", "worker") + assert not dc("ps", "--status", "running", "-q", "worker") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = str(UUID(attachment["id"])) + path = f"/api/v1/attachments/{attachment_id}" + + initial = job_details(attachment_id) + assert initial["attempt"] == 1 + assert initial["max_attempts"] == 5 + evidence["attachment_id"] = attachment_id + evidence["initial_job"] = initial + + restore_staging = True + remove_staging(attachment_id) + print("Test staging object removed; configured attempt limit=5") + + dc("start", "worker") + started = time.monotonic() + deadline = started + 180 + observed_attempts = [] + previous_attempt = None + + while time.monotonic() < deadline: + current = job_details(attachment_id) + if current["attempt"] != previous_attempt: + previous_attempt = current["attempt"] + observed_attempts.append(previous_attempt) + print( + f"Observed attempt={previous_attempt}, " + f"status={current['status']}", + flush=True, + ) + + if current["status"] == "failed": + break + if current["status"] == "ready": + raise AssertionError("Missing staging unexpectedly reached ready") + time.sleep(0.5) + else: + raise AssertionError("Retry exhaustion timeout") + + assert current["job_id"] == initial["job_id"] + assert current["attempt"] == current["max_attempts"] == 5 + assert current["error_code"] == "retry_exhausted" + assert current["lease_cleared"] is True + + outbox = current["outbox"] + assert [row["attempt"] for row in outbox] == [1, 2, 3, 4, 5] + assert all(row["published"] for row in outbox) + + for row in outbox: + if row["attempt"] == 1: + continue + previous = row["attempt"] - 1 + upper = min(30.0, 2.0 ** min(previous, 5)) + assert 0.49 <= row["delay_seconds"] <= upper + 0.1 + + failed = api("GET", path, 200, token=token) + assert failed["status"] == "failed" + assert failed["error_code"] == "retry_exhausted" + assert failed["variants"] == [] + api("GET", path + "/content", 409, token=token) + + count = int(sql(f""" + SELECT count(*) FROM attachment_variants + WHERE attachment_id = '{attachment_id}'::uuid; + """)) + assert count == 0 + + # Короткое дополнительное наблюдение терминального состояния. + time.sleep(5) + assert job_details(attachment_id) == current + + evidence["observed_attempts"] = observed_attempts + evidence["final_job"] = current + evidence["terminal_observation_seconds"] = 5 + evidence["elapsed_s"] = round(time.monotonic() - started, 3) + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + # Возвращаем принятые bytes, но failed автоматически не повторяем. + if restore_staging and attachment_id is not None: + staging_write(attachment_id, original=original) + finally: + try: + if restore_worker: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print( + "PASS: five published attempts exhausted retries; " + "backoff bounds verified; terminal failed/retry_exhausted, no variants" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_replacement.py b/student-tests/rolling_replacement.py new file mode 100644 index 00000000..9989eb4a --- /dev/null +++ b/student-tests/rolling_replacement.py @@ -0,0 +1,284 @@ +import json +from pathlib import Path +import subprocess +import sys +import time +from datetime import datetime, timezone + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, check, run + + +PROJECT = "mtusi2026lab3" + + +def timestamp(): + return datetime.now(timezone.utc).isoformat() + + +def command(*args): + return subprocess.check_output( + list(args), + cwd=ROOT, + text=True, + timeout=45, + ).strip() + + +def compose(*args): + return command( + "docker", + "compose", + "-p", + PROJECT, + *args, + ) + + +def inspect(container, template): + return command( + "docker", + "inspect", + "--format", + template, + container, + ) + + +def api_ids(): + ids = compose("ps", "-q", "api").split() + return { + inspect(container, "{{.Id}}") + for container in ids + } + + +def healthy(container): + return inspect( + container, + "{{.State.Running}} {{.State.Health.Status}}", + ) == "true healthy" + + +def instance_id(container): + code = ( + "import urllib.request; " + "r=urllib.request.urlopen(" + "'http://127.0.0.1:8080/health/live', timeout=2); " + "print(r.headers.get('X-Instance-Id', '')); " + "r.close()" + ) + return command( + "docker", + "exec", + container, + "python", + "-c", + code, + ) + + +def main(): + args = arguments("Sequential API container replacement") + + client = Client( + args.base_url, + args.ca_file, + timeout=2, + ) + alice, bob, eve, chat = client.bootstrap() + + import uuid + + key = uuid.uuid4().hex + text = "Сохранить при замене " + key + + original_message = client.message( + alice, + chat, + text, + key=key, + )["json"] + + original_ids = sorted(api_ids()) + check(len(original_ids) == 2, "expected exactly two API containers") + + for container in original_ids: + check(healthy(container), "both API containers must be healthy") + + print( + json.dumps( + { + "event": "replacement_started", + "utc": timestamp(), + "original_container_ids": original_ids, + }, + ensure_ascii=False, + ), + flush=True, + ) + + replacements = [] + + for old in original_ids: + current = api_ids() + check(len(current) == 2, "expected two APIs before replacement") + check(old in current, "original target container is missing") + + other = next(container for container in current if container != old) + check(healthy(other), "remaining API is not healthy") + + other_started = inspect(other, "{{.State.StartedAt}}") + old_instance = instance_id(old) + step_started = time.monotonic() + + print( + json.dumps( + { + "event": "replacing", + "utc": timestamp(), + "old_container_id": old, + "old_instance": old_instance, + "remaining_container_id": other, + }, + ensure_ascii=False, + ), + flush=True, + ) + + command("docker", "stop", "--time", "10", old) + command("docker", "rm", old) + + compose( + "up", + "-d", + "--no-deps", + "--no-recreate", + "--no-build", + "--scale", + "api=2", + "api", + ) + + deadline = time.monotonic() + 30 + replacement = None + + while time.monotonic() < deadline: + current = api_ids() + candidates = current - {other} + + if ( + len(current) == 2 + and len(candidates) == 1 + and other in current + ): + candidate = next(iter(candidates)) + if candidate != old and healthy(candidate): + replacement = candidate + break + + time.sleep(0.5) + + check(replacement is not None, "replacement did not become healthy") + + new_instance = instance_id(replacement) + check(new_instance != old_instance, "instance did not change") + + seen_through_lb = False + deadline = time.monotonic() + 20 + + while time.monotonic() < deadline: + try: + response = client.call( + "GET", + "/api/v1/users/me", + actor=alice, + schema="User", + ) + except Failure: + time.sleep(0.2) + continue + + if response["headers"]["X-Instance-Id"] == new_instance: + seen_through_lb = True + break + + time.sleep(0.2) + + check( + seen_through_lb, + "new API was not observed through LB with existing session", + ) + check(healthy(other), "remaining API became unhealthy") + check( + inspect(other, "{{.State.StartedAt}}") == other_started, + "the other API was restarted unexpectedly", + ) + + replay = client.message( + alice, + chat, + text, + key=key, + )["json"] + + check( + replay == original_message, + "confirmed message or idempotency result changed", + ) + + result = { + "event": "replacement_ready", + "utc": timestamp(), + "old_container_id": old, + "new_container_id": replacement, + "old_instance": old_instance, + "new_instance": new_instance, + "step_seconds": round( + time.monotonic() - step_started, 3 + ), + "other_api_not_restarted": True, + "existing_session_works": True, + "replay_preserved": True, + } + + replacements.append(result) + print(json.dumps(result, ensure_ascii=False), flush=True) + + time.sleep(3) + + final_ids = api_ids() + + check(len(final_ids) == 2, "expected two APIs after replacement") + check( + not final_ids.intersection(original_ids), + "some original API containers were not replaced", + ) + + for container in final_ids: + check(healthy(container), "final API is not healthy") + + print( + json.dumps( + { + "event": "replacement_finished", + "utc": timestamp(), + "replacements": len(replacements), + "final_container_ids": sorted(final_ids), + }, + ensure_ascii=False, + ), + flush=True, + ) + + print( + "PASS: both API containers replaced sequentially; " + "existing session and confirmed message preserved.", + flush=True, + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/staging_cleanup.py b/student-tests/staging_cleanup.py new file mode 100644 index 00000000..dcf0f16e --- /dev/null +++ b/student-tests/staging_cleanup.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +import base64 +import json +import secrets +import time +from uuid import UUID, uuid4 + +from queued_restart import ROOT, api, dc, http, wait_ready + + +EVIDENCE = ROOT / "evidence" / "staging_cleanup.json" + + +def storage_action(attachment_id, action): + code = """ +import json +import sys +from uuid import UUID + +from app.job_db import connect_db +from app.storage import _get, _put, bucket_name, get_client + +attachment_id = UUID(sys.argv[1]) +action = sys.argv[2] +prefix = f"staging/{attachment_id}/" +extra = prefix + "v99" +client = get_client() +bucket = bucket_name() + +if action == "add_orphan": + _put(extra, b"synthetic unreferenced staging object", "image/png") + +elif action == "remove_orphan": + client.delete_object(Bucket=bucket, Key=extra) + +elif action == "add_ready_leftover": + with connect_db() as db: + with db.transaction(): + row = db.execute( + ''' + SELECT object_key, content_type, status + FROM attachments + WHERE id = %s + FOR UPDATE + ''', + (attachment_id,), + ).fetchone() + if row is None or row["status"] != "ready": + raise RuntimeError("Expected a genuinely ready attachment") + data = _get(row["object_key"]) + staging_key = prefix + "v1" + _put(staging_key, data, row["content_type"]) + db.execute( + "UPDATE attachments SET staging_key = %s WHERE id = %s", + (staging_key, attachment_id), + ) + +elif action != "inspect": + raise ValueError("Unknown action") + +keys = [] +for page in client.get_paginator("list_objects_v2").paginate( + Bucket=bucket, Prefix=prefix, +): + keys.extend(item["Key"] for item in page.get("Contents", [])) + +with connect_db() as db: + row = db.execute( + ''' + SELECT status, staging_key IS NOT NULL AS staging_reference + FROM attachments WHERE id = %s + ''', + (attachment_id,), + ).fetchone() + +print(json.dumps({"keys": sorted(keys), **row})) +""" + return json.loads( + dc( + "exec", "-T", "api", + "python", "-c", code, attachment_id, action, + ) + ) + + +def cleanup(attachment_id, *, apply=False): + args = [ + "exec", "-T", "worker", + "python", "-m", "app.cleanup_staging", + "--attachment-id", attachment_id, + "--grace-seconds", "0", + "--allow-short-grace", + ] + if apply: + args.append("--apply") + return json.loads(dc(*args)) + + +def main(): + evidence = {"scenario": "scoped staging cleanup safety"} + attachment_id = None + restore = False + + try: + wait_ready() + if not dc("ps", "--status", "running", "-q", "worker"): + raise AssertionError("Start worker before this test") + + username = "cln_" + uuid4().hex[:16] + password = secrets.token_hex(16) + user = api( + "POST", "/api/v1/users", 201, + payload={ + "username": username, + "display_name": "Staging cleanup test", + "password": password, + }, + ) + basic = base64.b64encode( + f"{username}:{password}".encode() + ).decode() + token = api( + "POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + basic}, + )["token"] + + chat = api( + "POST", "/api/v1/chats", 201, + token=token, + payload={ + "title": "Staging cleanup", + "member_ids": [user["id"]], + }, + ) + + restore = True + dc("stop", "worker") + assert not dc("ps", "--status", "running", "-q", "worker") + + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment = api( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", + 202, + token=token, + data=original, + headers={"Content-Type": "image/png"}, + ) + attachment_id = str(UUID(attachment["id"])) + path = f"/api/v1/attachments/{attachment_id}" + active_key = f"staging/{attachment_id}/v1" + orphan_key = f"staging/{attachment_id}/v99" + + state = storage_action(attachment_id, "add_orphan") + assert state["status"] == "queued" + assert state["keys"] == sorted([active_key, orphan_key]) + + # worker остановлен, поэтому уборщик запускаем в API-контейнере. + # Модуль и настройки хранилищ у него те же. + def scoped_cleanup(apply): + args = [ + "exec", "-T", "api", + "python", "-m", "app.cleanup_staging", + "--attachment-id", attachment_id, + "--grace-seconds", "0", + "--allow-short-grace", + ] + if apply: + args.append("--apply") + return json.loads(dc(*args)) + + dry = scoped_cleanup(False) + assert dry["active"] == 1 + assert dry["candidates"] == 1 + assert dry["deleted"] == 0 + assert storage_action(attachment_id, "inspect")["keys"] == state["keys"] + + applied = scoped_cleanup(True) + assert applied["active"] == 1 + assert applied["deleted"] == 1 + + state = storage_action(attachment_id, "inspect") + assert state["keys"] == [active_key] + assert state["staging_reference"] is True + assert api("GET", path, 200, token=token)["status"] == "queued" + + evidence["queued_dry_run"] = dry + evidence["queued_apply"] = applied + print("Dry-run preserved both objects; apply removed only the orphan") + + dc("start", "worker") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + ready = api("GET", path, 200, token=token) + if ready["status"] == "ready": + break + if ready["status"] == "failed": + raise AssertionError("Active staging was not processed") + time.sleep(0.5) + else: + raise AssertionError("Ready timeout") + + # Дожидаемся завершения штатной уборки в job_runner. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + state = storage_action(attachment_id, "inspect") + if not state["keys"] and not state["staging_reference"]: + break + time.sleep(0.5) + else: + raise AssertionError("Normal staging cleanup did not finish") + + # Имитируем остаток после сбоя удаления, не подделывая ready: + # оригинал и варианты уже созданы настоящим worker. + storage_action(attachment_id, "add_ready_leftover") + + ready_dry = cleanup(attachment_id) + assert ready_dry["candidates"] == 1 + assert ready_dry["deleted"] == 0 + + ready_apply = cleanup(attachment_id, apply=True) + assert ready_apply["deleted"] == 1 + + state = storage_action(attachment_id, "inspect") + assert state["status"] == "ready" + assert state["keys"] == [] + assert state["staging_reference"] is False + + assert api("GET", path, 200, token=token) == ready + status, body = http("GET", path + "/content", token=token) + assert status == 200 and body == original + + for variant in ready["variants"]: + status, body = http( + "GET", path + "/content?variant=" + variant["name"], + token=token, + ) + assert status == 200 + assert len(body) == variant["size_bytes"] + + evidence["attachment_id"] = attachment_id + evidence["ready_dry_run"] = ready_dry + evidence["ready_apply"] = ready_apply + evidence["final_storage_state"] = state + evidence["result"] = "PASS" + + except BaseException as exc: + evidence["result"] = "FAIL" + evidence["error_type"] = type(exc).__name__ + raise + + finally: + try: + if attachment_id is not None: + storage_action(attachment_id, "remove_orphan") + finally: + try: + if restore: + dc("start", "worker") + finally: + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n" + ) + + print( + "PASS: dry-run is non-destructive; queued staging preserved; " + "orphan and ready leftover removed; ready content intact" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/targeted_failover.py b/student-tests/targeted_failover.py new file mode 100644 index 00000000..b002fb18 --- /dev/null +++ b/student-tests/targeted_failover.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Targeted API failure test. Uses synthetic data; leaves target stopped.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, arguments, check, run +from resilience import history + + +PROJECT = "mtusi2026lab3" + + +def docker_output(*args): + return subprocess.check_output( + ["docker", *args], + cwd=ROOT, + text=True, + ).strip() + + +def target_instance(container): + labels = json.loads( + docker_output( + "inspect", + "--format", + "{{json .Config.Labels}}", + container, + ) + ) + + check( + labels.get("com.docker.compose.project") == PROJECT, + "target belongs to another Compose project", + ) + check( + labels.get("com.docker.compose.service") == "api", + "target is not an API container", + ) + check( + docker_output( + "inspect", + "--format", + "{{.State.Running}}", + container, + ) == "true", + "target container is not running", + ) + + code = ( + "import urllib.request; " + "r=urllib.request.urlopen(" + "'http://127.0.0.1:8080/health/live', timeout=2); " + "print(r.headers.get('X-Instance-Id', '')); " + "r.close()" + ) + + instance = docker_output( + "exec", container, "python", "-c", code + ) + check(bool(instance), "target has no X-Instance-Id") + return instance + + +def main(): + def configure(parser): + parser.add_argument( + "--target-container", + required=True, + ) + + args = arguments( + "Targeted API failover through the load balancer", + configure, + ) + + script = ROOT / "scripts" / "stop-one.sh" + check( + script.is_file() and os.access(script, os.X_OK), + "stop-one.sh must exist and be executable", + ) + + target = target_instance(args.target_container) + client = Client(args.base_url, args.ca_file, timeout=2) + + alice, bob, eve, chat = client.bootstrap() + + original_key = uuid.uuid4().hex + original_text = "Подтверждено до отказа " + original_key + original = client.message( + alice, + chat, + original_text, + key=original_key, + )["json"] + + observed = set() + for _ in range(40): + response = client.call( + "GET", + "/api/v1/users/me", + actor=alice, + schema="User", + ) + observed.add(response["headers"]["X-Instance-Id"]) + + check(len(observed) >= 2, "less than two API instances") + check(target in observed, "target is not observed through LB") + + env = dict(os.environ) + env["TARGET_INSTANCE_ID"] = target + + print( + "Target container:", + args.target_container, + "| instance:", + target, + flush=True, + ) + + started = time.monotonic() + proc = subprocess.Popen( + [str(script)], + cwd=ROOT, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + successes = [] + write_failures = 0 + read_successes = 0 + read_failures = 0 + after_instances = [] + first_recovery = None + attempts = 0 + + key = uuid.uuid4().hex + text = "Во время отказа " + key + + try: + while time.monotonic() - started < 15: + code = proc.poll() + elapsed = time.monotonic() - started + + check( + code is None or code == 0, + "stop-one hook failed", + ) + check( + code is not None or elapsed < 10, + "stop-one hook exceeded 10 seconds", + ) + + attempts += 1 + + try: + response = client.message( + alice, + chat, + text, + key=key, + ) + except Failure: + write_failures += 1 + # Keep the same key and text after uncertain POST. + else: + successes.append(response["json"]) + + if proc.poll() == 0: + if first_recovery is None: + first_recovery = time.monotonic() - started + + after_instances.append( + response["headers"]["X-Instance-Id"] + ) + + key = uuid.uuid4().hex + text = "Во время отказа " + key + + if attempts % 5 == 0: + try: + page = client.call( + "GET", + ( + "/api/v1/chats/" + + chat["id"] + + "/messages?limit=100" + ), + actor=bob, + schema="MessagePage", + )["json"] + except Failure: + read_failures += 1 + else: + check( + any( + message == original + for message in page["items"] + ), + "confirmed message absent during read", + ) + read_successes += 1 + + time.sleep(0.1) + + elapsed = time.monotonic() - started + + check(proc.poll() == 0, "stop-one hook did not finish") + check( + first_recovery is not None and first_recovery <= 10, + "writes did not recover within 10 seconds", + ) + check( + len(after_instances) >= 5, + "too few writes after stop", + ) + check( + target not in after_instances[-5:], + "stopped instance still serves requests", + ) + check(read_successes > 0, "no successful history reads") + + check( + docker_output( + "inspect", + "--format", + "{{.State.Running}}", + args.target_container, + ) == "false", + "target container is still running", + ) + + replay = client.message( + alice, + chat, + original_text, + key=original_key, + )["json"] + + check( + replay == original, + "replay after failure returned another message", + ) + + messages = history(client, bob, chat) + by_id = {message["id"]: message for message in messages} + + check( + len(by_id) == len(messages), + "duplicate message IDs", + ) + check( + by_id.get(original["id"]) == original, + "pre-failure message lost or altered", + ) + + for message in successes: + check( + by_id.get(message["id"]) == message, + "confirmed in-flight message lost or altered", + ) + + texts = [message["text"] for message in messages] + check( + len(texts) == len(set(texts)), + "retry created duplicate message text", + ) + + print( + json.dumps( + { + "target_container": args.target_container, + "target_instance": target, + "observed_instances_before": len(observed), + "duration_seconds": round(elapsed, 3), + "successful_writes": len(successes), + "failed_write_attempts": write_failures, + "successful_reads": read_successes, + "failed_read_attempts": read_failures, + "successful_write_rps": round( + len(successes) / elapsed, 3 + ), + "first_recovery_seconds": round( + first_recovery, 3 + ), + "replay_preserved": True, + "target_stopped": True, + }, + ensure_ascii=False, + ) + ) + + print( + "PASS: targeted failover; session, messages " + "and replay preserved. Target remains stopped." + ) + + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/student-tests/test_cleanup_staging.py b/student-tests/test_cleanup_staging.py new file mode 100644 index 00000000..2bb9be45 --- /dev/null +++ b/student-tests/test_cleanup_staging.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import unittest +from unittest.mock import MagicMock, patch +from uuid import UUID + +from app.cleanup_staging import sweep + + +ATTACHMENT_ID = UUID("11111111-1111-4111-8111-111111111111") +KEY = f"staging/{ATTACHMENT_ID}/v1" + + +class CleanupStagingTests(unittest.TestCase): + def run_sweep( + self, + *, + status=None, + locked=True, + recent_failure=False, + recent_object=False, + apply=True, + ): + now = datetime.now(timezone.utc) + old = now - timedelta(days=2) + modified = now + timedelta(seconds=1) if recent_object else old + + row = None + if status is not None: + row = { + "id": ATTACHMENT_ID, + "status": status, + "updated_at": ( + now + timedelta(seconds=1) + if recent_failure else old + ), + } + + db = MagicMock() + db.__enter__.return_value = db + + def execute(query, params=None): + cursor = MagicMock() + if "pg_try_advisory_xact_lock" in query: + cursor.fetchone.return_value = {"locked": locked} + elif "SELECT id, status, updated_at" in query: + cursor.fetchone.return_value = row + return cursor + + db.execute.side_effect = execute + + client = MagicMock() + client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": KEY, "LastModified": modified}]} + ] + client.head_object.return_value = {"LastModified": modified} + + with ( + patch("app.cleanup_staging.connect_db", return_value=db), + patch("app.cleanup_staging.get_client", return_value=client), + patch("app.cleanup_staging.bucket_name", return_value="test-bucket"), + ): + result = sweep( + apply=apply, + grace_seconds=86400, + attachment_id=ATTACHMENT_ID, + ) + + return result, db, client + + def assert_no_delete(self, client): + client.delete_object.assert_not_called() + + def test_queued_is_preserved_even_when_old(self): + result, _, client = self.run_sweep(status="queued") + self.assertEqual(result["active"], 1) + self.assertEqual(result["deleted"], 0) + self.assert_no_delete(client) + + def test_processing_is_preserved_even_when_old(self): + result, _, client = self.run_sweep(status="processing") + self.assertEqual(result["active"], 1) + self.assertEqual(result["deleted"], 0) + self.assert_no_delete(client) + + def test_busy_upload_lock_prevents_deletion(self): + result, _, client = self.run_sweep(locked=False) + self.assertEqual(result["busy"], 1) + self.assert_no_delete(client) + client.head_object.assert_not_called() + + def test_recent_failed_state_preserves_old_object(self): + result, _, client = self.run_sweep( + status="failed", + recent_failure=True, + ) + self.assertEqual(result["fresh"], 1) + self.assert_no_delete(client) + + def test_recent_object_is_preserved(self): + result, _, client = self.run_sweep( + status="failed", + recent_object=True, + ) + self.assertEqual(result["fresh"], 1) + self.assert_no_delete(client) + + def test_expired_failed_staging_is_deleted_and_reference_cleared(self): + result, db, client = self.run_sweep(status="failed") + self.assertEqual(result["deleted"], 1) + client.delete_object.assert_called_once_with( + Bucket="test-bucket", + Key=KEY, + ) + updates = [ + call for call in db.execute.call_args_list + if "SET staging_key = NULL" in call.args[0] + ] + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].args[1], (ATTACHMENT_ID, KEY)) + + def test_expired_failed_dry_run_is_non_destructive(self): + result, db, client = self.run_sweep( + status="failed", + apply=False, + ) + self.assertEqual(result["candidates"], 1) + self.assertEqual(result["deleted"], 0) + self.assert_no_delete(client) + self.assertFalse(any( + "SET staging_key = NULL" in call.args[0] + for call in db.execute.call_args_list + )) + + def test_old_orphan_is_deleted_without_attachment_update(self): + result, db, client = self.run_sweep() + self.assertEqual(result["deleted"], 1) + client.delete_object.assert_called_once() + self.assertFalse(any( + "SET staging_key = NULL" in call.args[0] + for call in db.execute.call_args_list + )) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 00000000..763cc27a --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import hashlib +import io +import unittest + +from PIL import Image + +from app.image_pipeline import InvalidImage, render_variants + + +def make_image(size, *, mode="RGB", color="red", orientation=None): + with Image.new(mode, size, color) as image: + output = io.BytesIO() + if orientation is None: + image.save(output, format="PNG") + else: + exif = Image.Exif() + exif[274] = orientation + image.save(output, format="JPEG", exif=exif) + return output.getvalue() + + +def variants_by_name(data): + return { + variant["name"]: variant + for variant in render_variants(data) + } + + +class ImagePipelineTests(unittest.TestCase): + def assert_variant(self, variant, fmt, size): + data = variant["data"] + self.assertEqual(variant["size_bytes"], len(data)) + self.assertEqual( + variant["sha256"], + hashlib.sha256(data).hexdigest(), + ) + self.assertEqual( + (variant["width"], variant["height"]), + size, + ) + with Image.open(io.BytesIO(data)) as image: + image.load() + self.assertEqual(image.format, fmt) + self.assertEqual(image.size, size) + self.assertFalse(dict(image.getexif())) + self.assertFalse(image.info.get("icc_profile")) + self.assertFalse(image.info.get("xmp")) + + def test_contract_dimensions_and_original(self): + data = make_image((320, 200)) + before = hashlib.sha256(data).hexdigest() + variants = variants_by_name(data) + + self.assertEqual(set(variants), {"thumbnail", "crop", "optimized"}) + self.assert_variant(variants["thumbnail"], "JPEG", (256, 160)) + self.assert_variant(variants["crop"], "JPEG", (128, 128)) + self.assert_variant(variants["optimized"], "WEBP", (320, 200)) + self.assertEqual(hashlib.sha256(data).hexdigest(), before) + + def test_rounding_down(self): + variants = variants_by_name(make_image((321, 200))) + self.assert_variant(variants["thumbnail"], "JPEG", (256, 159)) + + def test_small_image_not_upscaled_except_crop(self): + variants = variants_by_name(make_image((64, 32))) + self.assert_variant(variants["thumbnail"], "JPEG", (64, 32)) + self.assert_variant(variants["optimized"], "WEBP", (64, 32)) + self.assert_variant(variants["crop"], "JPEG", (128, 128)) + + def test_transparency_becomes_white(self): + data = make_image((64, 32), mode="RGBA", color=(255, 0, 0, 0)) + variants = variants_by_name(data) + for variant in variants.values(): + with Image.open(io.BytesIO(variant["data"])) as image: + pixel = image.convert("RGB").getpixel((0, 0)) + self.assertTrue(all(channel >= 245 for channel in pixel)) + + def test_exif_orientation(self): + data = make_image((320, 200), orientation=6) + variants = variants_by_name(data) + self.assert_variant(variants["thumbnail"], "JPEG", (160, 256)) + self.assert_variant(variants["optimized"], "WEBP", (200, 320)) + + def test_invalid_input(self): + for data in (b"", b"not an image"): + with self.subTest(data=data): + with self.assertRaises(InvalidImage): + render_variants(data) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0