diff --git a/.dockerignore b/.dockerignore index bc16e556..9c4c0b6c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.superpowers +.pytest_cache diff --git a/.env.example b/.env.example index 119384b5..4b6b5538 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,5 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# bootstrap creates .env and private files in secrets/; never commit local values. SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +# Compose configures DATABASE_URL_FILE/CACHE_URL_FILE/S3_*_FILE for the application. +# Broker is separate from the optional profile cache; persistence AOF + appendfsync always. +# Processing limits: one worker, prefetch 1, decoder timeout 20s, lease 90s, 5 attempts. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..2d690d34 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + pipeline: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: make up + - run: make check validate-container test test-student test-fresh + - run: make test-jobs + - run: make test-cluster + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab5-evidence + path: evidence/ + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..74669a87 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.13.5-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt requirements-dev.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger +COPY src/ src/ +COPY migrations/ migrations/ +COPY student-tests/ student-tests/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +ARG APP_RELEASE=lab5-v1 +ENV APP_RELEASE=${APP_RELEASE} +LABEL org.opencontainers.image.version=${APP_RELEASE} +USER messenger +EXPOSE 8080 +CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"] diff --git a/Makefile b/Makefile index 5d86d171..aa202c79 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,38 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +.PHONY: bootstrap up down test-student test-cluster rolling +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + docker compose build api minio minio-init + docker compose up -d --wait --wait-timeout 120 + $(PYTHON) scripts/cluster.py wait +down: + docker compose down +test-student: + $(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +test-cluster: + $(PYTHON) scripts/verify_cluster.py +rolling: + $(PYTHON) scripts/cluster.py rolling + +.PHONY: test-fresh load-evidence verify-persistence verify-failover +test-fresh: + ./scripts/check_fresh_migrations.sh +load-evidence: + $(PYTHON) scripts/capture_load.py +verify-persistence: + $(PYTHON) scripts/verify_cluster.py persistence +verify-failover: + $(PYTHON) scripts/verify_cluster.py public-failover + $(PYTHON) scripts/verify_cluster.py failover + +.PHONY: validate-container +validate-container: + $(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py + + +.PHONY: test-jobs +test-jobs: + $(PYTHON) scripts/verify_jobs.py diff --git a/REPORT.md b/REPORT.md index e98f75e9..c05fc267 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,105 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Лукьянов К.Р. БПИ2303 +- Целевая оценка: **4**; трек 5 не заявлен. +- Commit/PR: заполняется автором после самостоятельного commit/push. +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/), [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose v2, Make, Python 3.9+ на хосте, свободный localhost:8080. Все Python-зависимости и Go для MinIO устанавливаются в сборочных контейнерах. Первая сборка требует доступа к registry, GitHub и пакетным репозиториям. Стенды предыдущих лабораторных следует остановить, сохранив volumes. + +```sh +git clone +cd +make up +curl -fsS http://localhost:8080/health/ready +make check +make validate-container +make test +make test-student +make test-fresh +make test-jobs +make test-cluster +``` + +`make up` создаёт локальные secrets, собирает API и MinIO/mc, запускает миграции и S3 bootstrap, затем две API, Nginx, отдельные broker/relay/worker. Версии образов и лимиты закреплены в compose.yaml, Python-зависимости — requirements.lock. MinIO и mc собираются из официальных release-исходников с проверкой commit: готовые образы/архив при выполнении lab4 были недоступны. Go на хосте не нужен. MinIO: RELEASE.2025-04-22T22-12-26Z, mc: RELEASE.2025-04-16T18-13-26Z; исходники и проверки — infra/minio/Dockerfile. + +`make down` сохраняет volumes. `docker compose down --volumes` удаляет все данные текущего учебного проекта. `make test-fresh` использует отдельный случайный Compose project и удаляет только его временные volumes. Сценарии отказов запускать последовательно, они меняют состояние контейнеров. Синтетические записи обычных тестов остаются в БД. + +Операторский replay без публичного административного endpoint: + +```sh +# job_id получить локально из attachment_jobs; токен пользователя не нужен. +docker compose exec -T worker python -m messenger.worker replay \ + --job-id --reason 'Причина после устранения неисправности' +``` + +Разрешён только failed с сохранённым staging. Ready replay отклоняется; audit пишется в job_replays. После retention источника replay явно запрещён. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент → Nginx → API×2 → MinIO staging + PostgreSQL metadata/job/outbox + ↓ relay + Redis Streams broker + ↓ worker + MinIO originals + thumbnail/crop/WebP → PG ready +``` + +Из lab4 перенесены API, auth/ACL, PostgreSQL, Redis-кэш профилей, Nginx, общие сессии, keyset и идемпотентность сообщений. Контракты и публичные tests текущей лабы сохранены. Объявленный переход: upload возвращает **202 queued**, content для queued/processing/failed — **409**, ready доступен через GET metadata. Сообщение может ссылаться на queued/processing, failed прикреплять нельзя. + +**Граница 202.** Тело ограничено 10 MiB и 40 млн пикселей. Проверяются MIME, структура и имя. Resize и полная декодировка выполняются worker. API сначала PUT в приватный `staging/`, затем в одной PG-транзакции создаёт attachment, job и outbox; только после COMMIT возвращает снимок queued. S3 и PG не образуют общей транзакции. Компенсация повторно проверяет metadata под lock, при недоступной БД слепого удаления нет. Неучтённый staging подберёт grace-cleanup. + +**Outbox.** Relay выбирает due записи под `FOR UPDATE SKIP LOCKED`, делает XADD и сохраняет dispatched_at. Сбой до XADD оставляет намерение; после XADD до COMMIT допускается дубль. Сообщение соответствует jobs.schema.json: постоянный job_id, attachment_id, schema/type, pipeline_version=1, attempt, created_at, опциональный W3C carrier. Байтов, storage URL и credentials в сообщении нет. + +**Устойчивость брокера.** Отдельный Redis 7.4.4 с volume, AOF и appendfsync always, noeviction. Redis-кэш профилей остаётся отдельным сервисом. Consumer group обеспечивает at least once; prefetch 1. XACK и затем XDEL выполняются после COMMIT результата либо устойчивого решения о retry/failed. Безусловного MAXLEN нет. Publisher acceptance не равен consumer ack. Успешный job остаётся в PG для проверки дублей. + +**Обработка и варианты.** Worker получает доверенные metadata из PG, сверяет staging SHA-256/размер. Детерминированные ключи оригинала и `(attachment,pipeline=1,variant)` исключают размножение вариантов. EXIF orientation применяется, alpha компонуется на белом, EXIF удаляется. Thumbnail: JPEG quality85, fit 256×256 без увеличения, floor/min1. Crop: центрированный квадрат 128×128 JPEG quality85, допускает увеличение. Optimized: WebP quality80, fit 1280×1280 без увеличения; уменьшение любого маленького файла не обещается. Оригинал побайтно сохраняется. Все варианты записываются до одной транзакции ready/checksums. + +**Попытки и блокировки.** PG session advisory lock удерживается через весь эффект. Lease 90s, attempt проверяется перед COMMIT. Relay восстанавливает expired processing только после получения того же lock; живой worker не вытесняется. SQL-ошибка сначала rollback, затем unlock; при неизвестном результате освобождения соединение закрывается, а не возвращается в pool. Дубль ready — no-op, старый attempt не расходует новую попытку. Постоянная ошибка изображения даёт failed, временная — retry. По умолчанию максимум 5 попыток, exponential backoff с jitter, верхняя граница задержки зависит от номера попытки и ограничена конфигурацией алгоритма. + +**Ограничение ресурсов.** Worker — 1 CPU/512 MiB, concurrency/prefetch 1. Весь цикл S3+decode+PUT выполняется в отдельной process group с абсолютным deadline 45s; supervisor убивает и дожидается всей группы до освобождения job-lock. Дочерний processor имеет собственный 45s alarm на случай смерти supervisor. Decoder дополнительно ограничен wall time 20s, CPU 15s и RLIMIT_AS 768 MiB; фактический общий container memory limit — 512 MiB. Временный каталог принадлежит supervisor и удаляется после завершения/timeout. Ограничения могут закончить особенно тяжёлый допустимый по пикселям файл контролируемым failed; unlimited RAM не обещается. + +**Очистка.** Staging удаляется после ready COMMIT; ключ сохраняется в metadata до успешного DeleteObject, relay повторяет очистку. Для failed источник хранится 24h, затем удаляется и replay запрещается. Grace 300s для неучтённых объектов; lock и свежая проверка PG сохраняют активные uploads, учтённые originals/variants/staging. БД недоступна — удаления нет. Частичные детерминированные варианты failed сохраняются до операторского решения; полного reconciler/retention всех вложений нет. + +MinIO приватен, console не опубликована. API/worker получают ограниченный S3 user для originals/staging/variants; root secrets только у MinIO/init. Единственный бизнес-порт — localhost:8080 Nginx. Брокер, API и хранилища не публикуют host ports. Приложение/worker non-root; secrets генерируются локально, не входят в Git/build context. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Прогоны 2026-11-09 на Docker Desktop/macOS Apple Silicon. Ветка `solution/bpi2303-lukyanov`; commits/push выполняет автор. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Проверка | Команда / доказательство | Результат | +| --- | --- | --- | +| Публичный контракт и API | make check, validate-container, test; evidence/check.txt, openapi.txt, smoke.txt | PASS | +| Регрессии, варианты, ACL, лимиты и SQL rollback | make test-student; evidence/student-tests.txt | 27 тестов PASS | +| Пустые volumes и повторные миграции 001–004 | make test-fresh; evidence/fresh-migrations.txt | PASS, 27 тестов | +| COMMIT без publish, worker stop, API/broker restart | make test-jobs; evidence/jobs-faults.json | та же принятая job → ready | +| Устойчивая непрочитанная очередь | evidence/jobs-faults.json | broker restart с volume, затем ready | +| PUT до COMMIT/ack и lease recovery | evidence/jobs-faults.json | S3-эффект есть до kill, та же job завершена со второй попытки | +| Дубль ready | собственный тест + evidence/jobs-faults.json | metadata неизменны, ready replay запрещён | +| Ограниченный retry/failed/replay | evidence/jobs-faults.json | 5 ошибок → retry_exhausted, failed attachment → 409, явный replay → ready | +| Session lock после SQL-ошибки | test_replay_sql_failure_releases_session_lock | независимая PG-сессия получает lock | +| Абсолютный timeout S3 | test_deadline_covers_slow_s3 | drip-feed не удерживает pipeline за deadline | +| Сохранённые restart/failover/rolling | make test-cluster; evidence/*.json | результаты отдельных сценариев в evidence | + +В pytest остаются предупреждения botocore об utcnow и диагностический Pillow warning теста pixel-bomb; assertions проходят. Публичные тесты не ослаблялись. RED перед реализацией: публичный lab5 smoke против lab4 ожидал 202 и получил 201 — evidence/lab4-baseline-red.txt. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`make test-jobs` реально останавливает relay/worker, загружает файл и создаёт сообщение, подтверждает 409 до готовности, перезапускает API и broker, затем запускает обработку. Outbox до восстановления имеет dispatched_at=null; новый job вручную не создаётся. Во втором опыте XADD уже выполнен, worker остановлен, broker перезапускается с volume: непрочитанная задача сохраняется. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Crash после side effect: BEFORE UPDATE trigger на конкретном attachment задерживает ready COMMIT через pg_sleep(2). После наблюдения PgSleep и существования оригинала в S3 worker получает SIGKILL. Trigger удаляется, lease истекает реально, без подмены часов. Зафиксировано восстановление примерно через **90.804s**, attempt=2, прежний job_id. Затем повтор ready возвращает no-op и прежние metadata. + +Для проверки предела повторов удаляется staging синтетической работы. Пять реальных попыток чтения S3 завершаются failed/retry_exhausted. В этом тесте next_at явно переносится на сейчас между попытками, чтобы отдельно проверить счётчик и финальное состояние; измерением реального времени backoff этот опыт не является. После восстановления исходных байтов операторский replay пишет audit, та же job доходит до ready. Failed-вложение нельзя привязать к сообщению. + +При code review найден и воспроизведён leak session-lock после SQL-ошибки в replay. Регрессия была RED; после rollback-before-unlock — GREEN. Второй RED→GREEN проверяет медленный HTTP/S3 поток, который не нарушает socket inactivity timeout: теперь общий deadline убивает обработку целиком. Это ограничивает зависший processing, а не просто прекращает ожидание фонового потока. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host, одна PG и один MinIO/broker; отказ диска/узла и backup/HA не покрываются. +- At least once и идемпотентный эффект не означают exactly once. Один pipeline_version=1; конкуренция версий — трек 5, не заявлен. +- Redis AOF always повышает устойчивость на данном volume, но не гарантирует переживание потери самого volume. Не применяется небезопасное trimming непрочитанных jobs. +- Непривязанные учтённые вложения и частичные конечные варианты failed не удаляются автоматически. Failed staging хранится 24h; после удаления источник нельзя восстановить replay. +- API принимает PNG/JPEG в заданных пределах; наиболее тяжёлый файл может получить failed из-за ограничений worker. HTTP/Basic разрешён только в локальном изолированном учебном стенде с синтетическими данными. +- В lab6 добавляются наблюдаемость и alerts, в lab7 — TLS/mTLS и OpenBao. Исследовательские треки 5 не выполнялись. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..1e614022 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,173 @@ +name: mtusi-lab5 +services: + postgres: + image: postgres:17.5-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + redis: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.25 + migrate: + image: mtusi-lab5-api:lab5 + command: [python, -m, messenger.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + restart: 'no' + mem_limit: 256m + cpus: 0.5 + api: + image: mtusi-lab5-api:lab5 + build: . + scale: 2 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + POOL_MAX_SIZE: '8' + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [front, data] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + stop_grace_period: 10s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"] + interval: 2s + timeout: 5s + retries: 30 + mem_limit: 512m + cpus: 1.0 + minio: + image: mtusi-lab5-minio:2025-04-22 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: minio + secrets: [minio_root_user, minio_root_password] + volumes: [minio_data:/data] + networks: [data] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 10s + mem_limit: 512m + cpus: 1.0 + minio-init: + image: mtusi-lab5-mc:2025-04-16 + build: + context: . + dockerfile: infra/minio/Dockerfile + target: mc + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key] + volumes: + - ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro + - ./infra/minio/api-policy.json:/config/api-policy.json:ro + networks: [data] + depends_on: + minio: {condition: service_healthy} + restart: 'no' + mem_limit: 128m + cpus: 0.25 + broker: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction] + volumes: [broker_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 160m + cpus: 0.5 + worker: &background + image: mtusi-lab5-api:lab5 + command: [python, -m, messenger.worker, worker] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://minio:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL: redis://broker:6379/0 + JOB_MAX_ATTEMPTS: '5' + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + broker: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + stop_grace_period: 45s + restart: unless-stopped + mem_limit: 512m + cpus: 1.0 + relay: + <<: *background + command: [python, -m, messenger.worker, relay] + mem_limit: 192m + cpus: 0.25 + lb: + image: nginx:1.28.0-alpine + volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro'] + networks: [front] + ports: ['127.0.0.1:8080:8080'] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 1s + timeout: 8s + retries: 30 + mem_limit: 128m + cpus: 0.5 +networks: + front: + data: + internal: true +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + minio_root_user: {file: ./secrets/minio_root_user} + minio_root_password: {file: ./secrets/minio_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} +volumes: + postgres_data: + redis_data: + minio_data: + broker_data: diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/concurrent-idempotency.json b/evidence/concurrent-idempotency.json new file mode 100644 index 00000000..3c1dc3d2 --- /dev/null +++ b/evidence/concurrent-idempotency.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "parallel_requests": 8, + "observed_instances": [ + "2836440f76e24486ab068d4c71498d15", + "77545b2c0222441193411736edbf54ff" + ], + "responses_201": 8, + "messages_created": 1, + "different_body_status": 409 +} diff --git a/evidence/dependency-failures.json b/evidence/dependency-failures.json new file mode 100644 index 00000000..3c56d2ec --- /dev/null +++ b/evidence/dependency-failures.json @@ -0,0 +1,20 @@ +{ + "result": "PASS", + "redis": { + "profile_status": 200, + "fallback_seconds": 2.5178, + "anonymous": 401, + "readiness": 200 + }, + "postgres": { + "authenticated_status": 503, + "response_seconds": 0.006, + "anonymous_status": 401, + "internal_api_liveness": [ + 200, + 200 + ], + "note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable." + }, + "same_session_after_recovery": true +} diff --git a/evidence/fresh-migrations.txt b/evidence/fresh-migrations.txt new file mode 100644 index 00000000..f7198377 --- /dev/null +++ b/evidence/fresh-migrations.txt @@ -0,0 +1,70 @@ +./scripts/check_fresh_migrations.sh + Volume mtusi-lab5-migration-check-53406_redis_data Creating + Network mtusi-lab5-migration-check-53406_data Creating + Volume mtusi-lab5-migration-check-53406_postgres_data Creating + Volume mtusi-lab5-migration-check-53406_minio_data Creating + Volume mtusi-lab5-migration-check-53406_redis_data Creating + Network mtusi-lab5-migration-check-53406_data Creating + Volume mtusi-lab5-migration-check-53406_postgres_data Creating + Volume mtusi-lab5-migration-check-53406_minio_data Creating + Volume mtusi-lab5-migration-check-53406_postgres_data Created + Volume mtusi-lab5-migration-check-53406_postgres_data Created + Volume mtusi-lab5-migration-check-53406_redis_data Created + Volume mtusi-lab5-migration-check-53406_redis_data Created + Volume mtusi-lab5-migration-check-53406_minio_data Created + Volume mtusi-lab5-migration-check-53406_minio_data Created + Network mtusi-lab5-migration-check-53406_data Created + Network mtusi-lab5-migration-check-53406_data Created + Container mtusi-lab5-migration-check-53406-redis-1 Creating + Container mtusi-lab5-migration-check-53406-minio-1 Creating + Container mtusi-lab5-migration-check-53406-postgres-1 Creating + Container mtusi-lab5-migration-check-53406-minio-1 Created + Container mtusi-lab5-migration-check-53406-minio-init-1 Creating + Container mtusi-lab5-migration-check-53406-redis-1 Created + Container mtusi-lab5-migration-check-53406-postgres-1 Created + Container mtusi-lab5-migration-check-53406-minio-init-1 Created + Container mtusi-lab5-migration-check-53406-minio-1 Starting + Container mtusi-lab5-migration-check-53406-postgres-1 Starting + Container mtusi-lab5-migration-check-53406-redis-1 Starting + Container mtusi-lab5-migration-check-53406-minio-1 Started + Container mtusi-lab5-migration-check-53406-minio-1 Waiting + Container mtusi-lab5-migration-check-53406-postgres-1 Started + Container mtusi-lab5-migration-check-53406-redis-1 Started + Container mtusi-lab5-migration-check-53406-minio-1 Healthy + Container mtusi-lab5-migration-check-53406-minio-init-1 Starting + Container mtusi-lab5-migration-check-53406-minio-init-1 Started + Container mtusi-lab5-migration-check-53406-redis-1 Waiting + Container mtusi-lab5-migration-check-53406-minio-1 Waiting + Container mtusi-lab5-migration-check-53406-postgres-1 Waiting + Container mtusi-lab5-migration-check-53406-minio-init-1 Waiting + Container mtusi-lab5-migration-check-53406-redis-1 Healthy + Container mtusi-lab5-migration-check-53406-minio-1 Healthy + Container mtusi-lab5-migration-check-53406-postgres-1 Healthy + Container mtusi-lab5-migration-check-53406-minio-init-1 Healthy + Network mtusi-lab5-migration-check-53406_front Creating + Network mtusi-lab5-migration-check-53406_front Creating + Network mtusi-lab5-migration-check-53406_front Created + Network mtusi-lab5-migration-check-53406_front Created + Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Creating + Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Created +Applied 001_initial.sql +Applied 002_idempotency.sql +Applied 003_attachments.sql +Applied 004_jobs.sql +Migrations up to date +Migrations up to date +........................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 19 warnings +student-tests/test_idempotency.py: 1 warning +student-tests/test_jobs.py: 6 warnings + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +27 passed, 27 warnings in 11.96s +PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes. diff --git a/evidence/graceful-shutdown.json b/evidence/graceful-shutdown.json new file mode 100644 index 00000000..4e86e810 --- /dev/null +++ b/evidence/graceful-shutdown.json @@ -0,0 +1,16 @@ +{ + "result": "PASS", + "target_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "target_instance": "3cd11c3ed6654d15aa331b17da172ba6", + "fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards", + "in_flight_observed_in_pg_stat_activity": true, + "in_flight_response": 201, + "new_tcp_after_sigterm": { + "accepted": false, + "error_type": "ConnectionRefusedError" + }, + "target_still_running_when_new_tcp_rejected": true, + "exit_code": 0, + "shutdown_seconds_from_sigterm": 2.1039, + "confirmed_message_preserved": true +} diff --git a/evidence/infrastructure.json b/evidence/infrastructure.json new file mode 100644 index 00000000..a2e664dc --- /dev/null +++ b/evidence/infrastructure.json @@ -0,0 +1,215 @@ +{ + "result": "PASS", + "containers": [ + { + "service": "api", + "container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e", + "name": "mtusi-lab5-api-3", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:16:43.516831177Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "f0356f65fd3349ebb6a12a12cbe8911b", + "release": "lab5-03545ea5" + }, + { + "service": "api", + "container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca", + "name": "mtusi-lab5-api-4", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:16:48.637814055Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "1f3ca0aa757548729c3af44cfe1152d8", + "release": "lab5-03545ea5" + }, + { + "service": "broker", + "container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646", + "name": "mtusi-lab5-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.613805961Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_broker_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "lb", + "container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732", + "name": "mtusi-lab5-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.692474795Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab5_front" + ] + }, + { + "service": "migrate", + "container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8", + "name": "mtusi-lab5-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:15:50.728753166Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio", + "container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6", + "name": "mtusi-lab5-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.65248217Z", + "image": "mtusi-lab5-minio:2025-04-22", + "image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_minio_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio-init", + "container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd", + "name": "mtusi-lab5-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:15:50.664761458Z", + "image": "mtusi-lab5-mc:2025-04-16", + "image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "postgres", + "container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3", + "name": "mtusi-lab5-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:17:03.314404256Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_postgres_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "redis", + "container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3", + "name": "mtusi-lab5-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:17:02.234800422Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_redis_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "relay", + "container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce", + "name": "mtusi-lab5-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:15:28.180030295Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "worker", + "container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5", + "name": "mtusi-lab5-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:15:28.686125545Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + } + ] +} diff --git a/evidence/jobs-faults.json b/evidence/jobs-faults.json new file mode 100644 index 00000000..45229d3c --- /dev/null +++ b/evidence/jobs-faults.json @@ -0,0 +1,27 @@ +{ + "result": "PASS", + "commit_publish_gap": { + "accepted_status": 202, + "outbox_unpublished": true, + "api_broker_restarted": true, + "same_job_ready": true + }, + "broker_persistence": { + "unread_job_survived_restart": true + }, + "after_put_before_ack": { + "s3_effect_written": true, + "lease_recovery_seconds": 90.804, + "attempt_after_recovery": 2, + "same_job": true + }, + "duplicate": { + "ready_noop": true + }, + "retry_replay": { + "failed_after": 5, + "code": "retry_exhausted", + "explicit_replay_ready": true, + "failed_attachment_binding": 409 + } +} diff --git a/evidence/lab4-baseline-red.txt b/evidence/lab4-baseline-red.txt new file mode 100644 index 00000000..cac79756 --- /dev/null +++ b/evidence/lab4-baseline-red.txt @@ -0,0 +1 @@ +FAIL: POST /api/v1/chats/2a81dc08-f62f-4897-899d-7c925a255fbf/attachments: expected [202], received 201 diff --git a/evidence/last-stop.json b/evidence/last-stop.json new file mode 100644 index 00000000..976f5d15 --- /dev/null +++ b/evidence/last-stop.json @@ -0,0 +1,11 @@ +{ + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "instance": "fa72717b0a734be9b12e177fd2ca606d", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "stopped_at_utc": "2026-11-09T15:16:14.991795+00:00", + "stop_seconds": 0.5132, + "exit_code": 0, + "left_stopped": true +} diff --git a/evidence/nginx.json b/evidence/nginx.json new file mode 100644 index 00000000..dfcf8cbb --- /dev/null +++ b/evidence/nginx.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "version": "nginx version: nginx/1.28.0", + "image": "nginx:1.28.0-alpine", + "public_server_header": "nginx", + "config_syntax": "PASS", + "observed_instances": [ + "2836440f76e24486ab068d4c71498d15", + "77545b2c0222441193411736edbf54ff" + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..49c347b6 --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1,2 @@ +python3 scripts/cluster.py exec python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..14bc408d --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,428 @@ +{ + "result": "PASS", + "before": [ + { + "service": "api", + "container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "name": "mtusi-lab5-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:12:16.797696721Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "77545b2c0222441193411736edbf54ff", + "release": "lab5-v1" + }, + { + "service": "api", + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:12:16.716356512Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "2836440f76e24486ab068d4c71498d15", + "release": "lab5-v1" + }, + { + "service": "broker", + "container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646", + "name": "mtusi-lab5-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:12:27.23476842Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_broker_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "lb", + "container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732", + "name": "mtusi-lab5-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:06:24.855529918Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab5_front" + ] + }, + { + "service": "migrate", + "container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8", + "name": "mtusi-lab5-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:14:23.843447751Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio", + "container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6", + "name": "mtusi-lab5-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:06:18.468961513Z", + "image": "mtusi-lab5-minio:2025-04-22", + "image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_minio_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio-init", + "container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd", + "name": "mtusi-lab5-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:14:23.912373126Z", + "image": "mtusi-lab5-mc:2025-04-16", + "image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "postgres", + "container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3", + "name": "mtusi-lab5-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:06:18.410650346Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_postgres_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "redis", + "container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3", + "name": "mtusi-lab5-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:06:18.524762596Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_redis_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "relay", + "container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce", + "name": "mtusi-lab5-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:14:20.333630291Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "worker", + "container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5", + "name": "mtusi-lab5-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:14:20.398521Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + } + ], + "after": [ + { + "service": "api", + "container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "name": "mtusi-lab5-api-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.966931253Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "cda72c7a2f684f4f83c4f124fe97d9a6", + "release": "lab5-v1" + }, + { + "service": "api", + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.899393962Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data", + "mtusi-lab5_front" + ], + "http_status": 200, + "instance": "c306444bf8ac4a599b4dc6ac465b038a", + "release": "lab5-v1" + }, + { + "service": "broker", + "container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646", + "name": "mtusi-lab5-broker-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.613805961Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 167772160, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_broker_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "lb", + "container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732", + "name": "mtusi-lab5-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.692474795Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab5_front" + ] + }, + { + "service": "migrate", + "container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8", + "name": "mtusi-lab5-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:15:29.872387004Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio", + "container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6", + "name": "mtusi-lab5-minio-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.65248217Z", + "image": "mtusi-lab5-minio:2025-04-22", + "image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_minio_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "minio-init", + "container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd", + "name": "mtusi-lab5-minio-init-1", + "status": "exited", + "health": null, + "started_at": "2026-11-09T15:15:30.375772004Z", + "image": "mtusi-lab5-mc:2025-04-16", + "image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "postgres", + "container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3", + "name": "mtusi-lab5-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.672883961Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_postgres_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "redis", + "container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3", + "name": "mtusi-lab5-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.736927712Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab5_redis_data" + ], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "relay", + "container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce", + "name": "mtusi-lab5-relay-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:15:28.180030295Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 201326592, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + }, + { + "service": "worker", + "container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5", + "name": "mtusi-lab5-worker-1", + "status": "running", + "health": null, + "started_at": "2026-11-09T15:15:28.686125545Z", + "image": "mtusi-lab5-api:lab5", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab5_data" + ] + } + ], + "public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover.json b/evidence/public-failover.json new file mode 100644 index 00000000..bf0f74cb --- /dev/null +++ b/evidence/public-failover.json @@ -0,0 +1,15 @@ +{ + "result": "PASS", + "public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 133, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.878}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "stopped": { + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "instance": "c306444bf8ac4a599b4dc6ac465b038a", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "stopped_at_utc": "2026-11-09T15:15:35.619684+00:00", + "stop_seconds": 0.3357, + "exit_code": 0, + "left_stopped": true + } +} diff --git a/evidence/public-failover.txt b/evidence/public-failover.txt new file mode 100644 index 00000000..cbe4980f --- /dev/null +++ b/evidence/public-failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 0.878} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..8061d34a --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,171 @@ +{ + "result": "PASS", + "before": [ + { + "container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "name": "mtusi-lab5-api-1", + "instance": "d23b5da28be949239c1ea5f27a578822", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b" + }, + { + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "instance": "7c84431c2a104eba8f1f3d789d971602", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b" + } + ], + "after": [ + { + "container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e", + "name": "mtusi-lab5-api-3", + "instance": "f0356f65fd3349ebb6a12a12cbe8911b", + "release": "lab5-03545ea5", + "image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe" + }, + { + "container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca", + "name": "mtusi-lab5-api-4", + "instance": "1f3ca0aa757548729c3af44cfe1152d8", + "release": "lab5-03545ea5", + "image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe" + } + ], + "target_release": "lab5-03545ea5", + "lb_before": { + "service": "lb", + "container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732", + "name": "mtusi-lab5-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.692474795Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab5_front" + ] + }, + "lb_after": { + "service": "lb", + "container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732", + "name": "mtusi-lab5-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-09T15:15:27.692474795Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab5_front" + ] + }, + "lb_not_restarted": true, + "build_seconds": 3.8455, + "migration_output": "Migrations up to date", + "replacements": [ + { + "old_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "new_container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e", + "old_ip": "172.26.0.3", + "new_ip": "172.26.0.5", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "7c84431c2a104eba8f1f3d789d971602", + "d23b5da28be949239c1ea5f27a578822", + "f0356f65fd3349ebb6a12a12cbe8911b" + ], + "observed_instances": [ + "7c84431c2a104eba8f1f3d789d971602", + "f0356f65fd3349ebb6a12a12cbe8911b" + ], + "old_exit_code": 0, + "seconds": 5.1191 + }, + { + "old_container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "new_container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca", + "old_ip": "172.26.0.2", + "new_ip": "172.26.0.3", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "1f3ca0aa757548729c3af44cfe1152d8", + "7c84431c2a104eba8f1f3d789d971602", + "f0356f65fd3349ebb6a12a12cbe8911b" + ], + "observed_instances": [ + "1f3ca0aa757548729c3af44cfe1152d8", + "f0356f65fd3349ebb6a12a12cbe8911b" + ], + "old_exit_code": 0, + "seconds": 6.1668 + } + ], + "during_replacement": { + "seconds": 11.286, + "concurrency": 4, + "successful_requests": 8541, + "failed_attempts": 0, + "successful_rps": 756.778, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.383, + "p95": 7.793, + "p99": 35.123 + }, + "instances": [ + "1f3ca0aa757548729c3af44cfe1152d8", + "7c84431c2a104eba8f1f3d789d971602", + "d23b5da28be949239c1ea5f27a578822", + "f0356f65fd3349ebb6a12a12cbe8911b" + ] + }, + "total": { + "seconds": 22.0756, + "concurrency": 4, + "successful_requests": 16487, + "failed_attempts": 0, + "successful_rps": 746.843, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.601, + "p95": 7.648, + "p99": 26.561 + }, + "instances": [ + "1f3ca0aa757548729c3af44cfe1152d8", + "7c84431c2a104eba8f1f3d789d971602", + "d23b5da28be949239c1ea5f27a578822", + "f0356f65fd3349ebb6a12a12cbe8911b" + ] + }, + "data": { + "confirmed_unique_messages": 16487, + "history_messages": 16487, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate." +} diff --git a/evidence/sequential-failover.json b/evidence/sequential-failover.json new file mode 100644 index 00000000..25639554 --- /dev/null +++ b/evidence/sequential-failover.json @@ -0,0 +1,135 @@ +{ + "result": "PASS", + "baseline": { + "seconds": 3.0041, + "concurrency": 4, + "successful_requests": 2367, + "failed_attempts": 0, + "successful_rps": 787.933, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.168, + "p95": 6.086, + "p99": 33.522 + }, + "instances": [ + "cda72c7a2f684f4f83c4f124fe97d9a6", + "fa72717b0a734be9b12e177fd2ca606d" + ] + }, + "phases": [ + { + "stopped": { + "container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808", + "name": "mtusi-lab5-api-1", + "instance": "cda72c7a2f684f4f83c4f124fe97d9a6", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "stopped_at_utc": "2026-11-09T15:16:01.326356+00:00", + "stop_seconds": 0.4399, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0034, + "concurrency": 4, + "successful_requests": 4111, + "failed_attempts": 1, + "successful_rps": 513.658, + "error_ratio": 0.000243, + "successful_latency_ms": { + "p50": 6.469, + "p95": 12.349, + "p99": 58.778 + }, + "instances": [ + "cda72c7a2f684f4f83c4f124fe97d9a6", + "fa72717b0a734be9b12e177fd2ca606d" + ] + }, + "post_stop_write_seconds_from_action_start": 1.4598, + "post_stop_write_seconds_from_stop_completion": 0.029, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.4513, + "seconds_from_stop_completion": 0.0206 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + }, + { + "stopped": { + "container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6", + "name": "mtusi-lab5-api-2", + "instance": "fa72717b0a734be9b12e177fd2ca606d", + "release": "lab5-v1", + "image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b", + "stopped_at_utc": "2026-11-09T15:16:14.991795+00:00", + "stop_seconds": 0.5132, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0007, + "concurrency": 4, + "successful_requests": 4087, + "failed_attempts": 0, + "successful_rps": 510.829, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 6.497, + "p95": 13.243, + "p99": 57.218 + }, + "instances": [ + "3cd11c3ed6654d15aa331b17da172ba6", + "fa72717b0a734be9b12e177fd2ca606d" + ] + }, + "post_stop_write_seconds_from_action_start": 1.3033, + "post_stop_write_seconds_from_stop_completion": 0.0159, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.3004, + "seconds_from_stop_completion": 0.013 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + } + ], + "total": { + "seconds": 30.9184, + "concurrency": 4, + "successful_requests": 18469, + "failed_attempts": 1, + "successful_rps": 597.347, + "error_ratio": 5.4e-05, + "successful_latency_ms": { + "p50": 5.53, + "p95": 10.788, + "p99": 46.459 + }, + "instances": [ + "3cd11c3ed6654d15aa331b17da172ba6", + "7c84431c2a104eba8f1f3d789d971602", + "cda72c7a2f684f4f83c4f124fe97d9a6", + "fa72717b0a734be9b12e177fd2ca606d" + ] + }, + "data": { + "confirmed_unique_messages": 18469, + "history_messages": 18469, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "each_original_container_tested": true +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..df05fe26 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..02a8c464 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,15 @@ +python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +........................... [100%] +=============================== warnings summary =============================== +student-tests/test_attachments.py: 19 warnings +student-tests/test_idempotency.py: 1 warning +student-tests/test_jobs.py: 6 warnings + /usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() + +student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename + /usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +27 passed, 27 warnings in 11.58s diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..d1a5fca4 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,28 @@ +# Official public images/binaries were removed upstream; build fixed release sources. +FROM golang:1.24.2-bookworm AS minio-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio . +RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio . + +FROM golang:1.24.2-bookworm AS mc-build +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local +RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src +WORKDIR /src +RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc . +RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc . + +FROM python:3.13.5-slim AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z" +RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data +COPY --from=minio-build /out/minio /usr/local/bin/minio +COPY infra/minio/server.sh /server.sh +USER 10002 +ENTRYPOINT ["/bin/sh", "/server.sh"] +CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"] + +FROM debian:bookworm-slim AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z" +COPY --from=mc-build /out/mc /usr/local/bin/mc +ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"] diff --git a/infra/minio/api-policy.json b/infra/minio/api-policy.json new file mode 100644 index 00000000..bd561fd7 --- /dev/null +++ b/infra/minio/api-policy.json @@ -0,0 +1,45 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments" + ], + "Condition": { + "StringLike": { + "s3:prefix": [ + "originals/*", + "staging/*", + "variants/*" + ] + } + } + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + } + ] +} diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..7d9eb39d --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy create local messenger-api /config/api-policy.json >/dev/null +mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).' diff --git a/infra/minio/server.sh b/infra/minio/server.sh new file mode 100644 index 00000000..aedb392b --- /dev/null +++ b/infra/minio/server.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +exec minio "$@" diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..5f0b43d3 --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,62 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log off; + server_tokens off; + limit_conn_zone $server_name zone=uploads:64k; + + # Docker DNS; resolve updates the upstream when replicas get new IPs. + resolver 127.0.0.11 valid=1s ipv6=off; + resolver_timeout 1s; + + upstream messenger { + zone messenger 64k; + # All A records of api participate in the default round-robin balancing. + server api:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + server_name messenger; + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + limit_conn uploads 16; + client_max_body_size 11m; + client_body_buffer_size 64k; + client_body_timeout 10s; + proxy_request_buffering on; + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + } + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + + # Passive checks. Do NOT enable non_idempotent: an already-sent POST + # must not be resent by Nginx after an ambiguous response. + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 6s; + } + } +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..7b8c5338 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..855e66a8 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +-- Additive: old lab2 tables and messages remain valid. +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a3de2f85 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,23 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id,id); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id,attachment_id), + UNIQUE (message_id,position) +); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 00000000..51f3c121 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1), + attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0, + lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text +); +CREATE TABLE job_outbox ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz +); +CREATE TABLE job_replays ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL +); +CREATE INDEX jobs_due ON attachment_jobs(next_at); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..78da0202 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,51 @@ +annotated-types==0.8.0 +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +attrs==26.1.0 +boto3==1.38.27 +botocore==1.38.46 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +iniconfig==2.3.1 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +packaging==26.3 +pathable==0.4.4 +pillow==11.3.0 +pluggy==1.7.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.21.0 +pytest==8.4.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.3 +redis==6.2.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +s3transfer==0.13.1 +six==1.17.0 +sniffio==1.3.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d7c23069 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +argon2-cffi==25.1.0 +pytest==8.4.0 +httpx==0.28.1 +pydantic==2.11.5 +anyio==4.9.0 +boto3==1.38.27 +Pillow==11.3.0 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..010bc7ce --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +import secrets +root = Path('secrets') +p = root / 'postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +pw = p.read_text().strip() +for name, value in { + 'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger', + 'cache_url': 'redis://redis:6379/0', + 'minio_root_user': 'lab4-admin', + 'minio_root_password': secrets.token_hex(24), + 's3_access_key_id': 'messenger-api', + 's3_secret_access_key': secrets.token_hex(24), +}.items(): + path = root / name + if not path.exists(): + path.write_text(value) +# Bind-mounted Docker secrets must be readable by container uid 10001. +# Parent directory remains 0700 on the host. +for path in root.iterdir(): + path.chmod(0o444) +if not Path('.env').exists(): + Path('.env').write_text('SESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden).') +PY diff --git a/scripts/check_fresh_migrations.sh b/scripts/check_fresh_migrations.sh new file mode 100755 index 00000000..c6112273 --- /dev/null +++ b/scripts/check_fresh_migrations.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A unique disposable Compose project; never touches normal project volumes. +set -eu +cd "$(dirname "$0")/.." +project="mtusi-lab5-migration-check-$$" +if [ -n "$(docker compose -p "$project" ps -aq)" ]; then + echo 'Refusing to reuse a pre-existing scratch project' >&2 + exit 1 +fi +cleanup() { + docker compose -p "$project" down --volumes >/dev/null 2>&1 +} +trap cleanup EXIT INT TERM +docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init +docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache' +echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.' diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 00000000..34de6929 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence.""" +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') + + +def command(args, timeout=120): + return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip() + + +def compose(*args, timeout=120): + return command(['docker', 'compose', *args], timeout=timeout) + + +def inspect(containers): + if not containers: + return [] + return json.loads(command(['docker','inspect',*containers])) + + +def container_ids(service=None, all_containers=False): + args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else []) + return compose(*args).split() + + +def probe(container, host='127.0.0.1', path='/health/live'): + code = '''import json,sys,urllib.request,urllib.error +try: + r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4) +except urllib.error.HTTPError as exc: + r=exc +print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')})) +''' + return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8)) + + +def replicas(): + items = [] + for data in inspect(container_ids('api')): + if data['State']['Running']: + result = probe(data['Id']) + items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']}) + return items + + +def snapshot(): + items = [] + for data in inspect(container_ids(all_containers=True)): + service = data['Config']['Labels']['com.docker.compose.service'] + item = { + 'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'), + 'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'], + 'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'], + 'published_ports':data['HostConfig']['PortBindings'], + 'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'), + 'networks':sorted(data['NetworkSettings']['Networks']), + } + if service == 'api' and data['State']['Running']: + result = probe(data['Id']) + item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']}) + items.append(item) + return items + + +def wait_healthy(container, timeout=60): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + data = inspect([container])[0] + if data['State'].get('Health',{}).get('Status') == 'healthy': + return + time.sleep(0.3) + raise RuntimeError('Container did not become healthy') + + +def wait_cluster(count=2, timeout=60): + deadline = time.monotonic()+timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response: + alias = response.headers.get('X-Instance-Id') + if alias: + seen.add(alias) + if len(seen) >= count: + return sorted(seen) + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise RuntimeError('Load balancer did not discover required API instances') + + +def stop_one(target): + if not re.fullmatch(r'[a-f0-9]{32}', target): + raise ValueError('Invalid target instance alias') + matching = [item for item in replicas() if item['instance']==target] + if len(matching) != 1: + raise RuntimeError('Target must match exactly one running API process') + item = matching[0] + started = time.monotonic() + command(['docker','stop','--time','7',item['container']],timeout=9) + state = inspect([item['container']])[0]['State'] + if state['Running'] or state['ExitCode'] != 0: + raise RuntimeError('API did not stop gracefully') + result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True} + (ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n') + return result + + +def restart(): + compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30) + compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40) + return wait_cluster(2,timeout=15) + + +def front_address(container): + networks = inspect([container])[0]['NetworkSettings']['Networks'] + return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front')) + + +def replace_one(container): + before = set(container_ids('api')) + if container not in before or len(before) != 2: + raise RuntimeError('Replacement expects exactly two running API containers') + old_address = front_address(container) + # Start-first ensures a genuinely different IP while the old address is still occupied. + compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api') + expanded = set(container_ids('api')) + added = expanded-before + if len(added) != 1 or not before.issubset(expanded): + raise RuntimeError('Start-first step unexpectedly changed an existing container') + replacement = added.pop() + wait_healthy(replacement) + new_address = front_address(replacement) + if new_address == old_address: + raise RuntimeError('Replacement did not get a different IP') + expected_three = {item['instance'] for item in replicas()} + if set(wait_cluster(3)) != expected_three: + raise RuntimeError('Load balancer did not discover all three live addresses') + command(['docker','stop','--time','7',container],timeout=10) + state = inspect([container])[0]['State'] + if state['ExitCode'] != 0: + raise RuntimeError('Old release did not terminate gracefully') + command(['docker','rm',container]) + compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api') + after = set(container_ids('api')) + if after != (before-{container}) | {replacement}: + raise RuntimeError('Sequential replacement changed the wrong containers') + expected = {item['instance'] for item in replicas()} + if set(wait_cluster(2)) != expected: + raise RuntimeError('Unexpected final backend aliases') + return {'old_container':container,'new_container':replacement, + 'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True, + 'instances_during_start_first':sorted(expected_three), + 'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec']) + parser.add_argument('arguments',nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.action == 'wait': + print(json.dumps({'instances':wait_cluster()})) + elif args.action == 'snapshot': + print(json.dumps(snapshot(),indent=2)) + elif args.action == 'stop-one': + print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID']))) + elif args.action == 'exec': + ids = container_ids('api') + if not ids or not args.arguments: + raise RuntimeError('A live API container and command are required') + raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT)) + elif args.action == 'restart': + print(json.dumps({'instances':restart()})) + else: + compose('run','--rm','--no-deps','migrate') + print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..3985fa9c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/cluster.py restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..ef14b25f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +exec python3 scripts/cluster.py stop-one diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py new file mode 100644 index 00000000..5eef4e22 --- /dev/null +++ b/scripts/verify_cluster.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts.""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +import uuid + +import cluster + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from resilience import history + + +def save(name, value): + (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') + print('PASS:', name) + + +def percentiles(values): + values = sorted(values) + return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None + for name,p in [('p50',.50),('p95',.95),('p99',.99)]} + + +class Traffic: + def __init__(self, concurrency=4): + self.c = Client(timeout=6) + self.alice,self.bob,_,self.chat = self.c.bootstrap() + self.concurrency = concurrency + self.done = threading.Event() + self.lock = threading.Lock() + self.attempts = [] + self.confirmed = {} + self.prefix = uuid.uuid4().hex + + def worker(self, number): + client = Client(timeout=6) + index = 0 + while not self.done.is_set(): + text = self.prefix+'-'+str(number)+'-'+str(index) + key = self.prefix+'-'+str(number)+'-'+str(index) + started = time.monotonic() + try: + response = client.message(self.alice,self.chat,text,key=key) + row = response['json'] + attempt = {'started':started,'ended':time.monotonic(),'ok':True, + 'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']} + with self.lock: + self.confirmed[row['id']] = row + index += 1 + except Failure: + # Only idempotent message POST is retried, with its original body and key. + attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None} + time.sleep(0.02) + with self.lock: + self.attempts.append(attempt) + + def start(self): + self.started = time.monotonic() + self.pool = ThreadPoolExecutor(max_workers=self.concurrency) + self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)] + return self + + def stop(self): + self.done.set() + for future in self.futures: + future.result(timeout=10) + self.pool.shutdown() + self.ended = time.monotonic() + + def summary(self, start=None, end=None): + start = self.started if start is None else start + end = getattr(self,'ended',time.monotonic()) if end is None else end + with self.lock: + attempts = [a for a in self.attempts if start <= a['started'] < end] + success = [a for a in attempts if a['ok']] + duration = end-start + return {'seconds':round(duration,4),'concurrency':self.concurrency, + 'successful_requests':len(success),'failed_attempts':len(attempts)-len(success), + 'successful_rps':round(len(success)/duration,3), + 'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0, + 'successful_latency_ms':percentiles([a['latency'] for a in success]), + 'instances':sorted({a['instance'] for a in success})} + + def verify_data(self): + rows = history(self.c,self.bob,self.chat) + actual = {row['id']:row for row in rows} + check(len(actual)==len(rows),'duplicated message ids') + check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message') + check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed') + return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows), + 'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0} + + +def concurrent_replay(): + cluster.wait_cluster() + c = Client() + alice,bob,_,chat = c.bootstrap() + key = uuid.uuid4().hex + barrier = threading.Barrier(8) + def send(_): + local = Client() + barrier.wait() + return local.message(alice,chat,'Один результат через разные процессы',key=key) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send,range(8))) + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(instances)>=2,'concurrent retry did not reach different API processes') + check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice, + body={'text':'Другой body'},headers={'Idempotency-Key':key}) + check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates') + save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances), + 'responses_201':8,'messages_created':1,'different_body_status':409}) + + +def verify_persistence(): + before = cluster.snapshot() + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=190) + after = cluster.snapshot() + roles = ['api','postgres','redis','minio','broker','relay','worker','lb'] + previous = {item['container']:item for item in before} + for item in after: + if item['service'] in roles: + old = previous[item['container']] + check(old['started_at']!=item['started_at'],'service did not restart: '+item['service']) + check(old['volumes']==item['volumes'],'volume identities changed') + (ROOT/'evidence/persistence.txt').write_text(result.stdout) + save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()}) + + +def public_failover(): + cluster.wait_cluster() + try: + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=100) + (ROOT/'evidence/public-failover.txt').write_text(result.stdout) + save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(), + 'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())}) + finally: + cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90') + cluster.wait_cluster() + + +def reject_new_connection(survivor, address): + code = '''import json,socket,sys +try: + s=socket.create_connection((sys.argv[1],8080),timeout=1) +except OSError as exc: + print(json.dumps({'accepted':False,'error_type':type(exc).__name__})) +else: + s.close() + print(json.dumps({'accepted':True})) +''' + return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address])) + + +def failover_each(): + cluster.wait_cluster() + originals = cluster.replicas() + check(len(originals)==2,'two running replicas required') + traffic = Traffic().start() + phases = [] + try: + time.sleep(3) + baseline_end = time.monotonic() + baseline = traffic.summary(end=baseline_end) + for original in originals: + # Alias changes on restart, so resolve the current process of this exact container. + current = next(r for r in cluster.replicas() if r['container']==original['container']) + all_before = cluster.inspect([current['container']])[0] + address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front')) + survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container']) + read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1' + marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0] + start = time.monotonic() + try: + env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']} + subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10) + stop_returned = time.monotonic() + stopped = json.loads((ROOT/'evidence/last-stop.json').read_text()) + read_deadline = start+10 + while True: + try: + read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage') + check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage') + check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process') + read_done = time.monotonic() + break + except Failure: + check(time.monotonic()=start] + recent = [a for a in successful if a['started']>stop_returned] + recovery = min(a['ended'] for a in recent)-start if recent else None + after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None + check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start') + check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests') + phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4), + 'post_stop_write_seconds_from_stop_completion':round(after_stop,4), + 'read_during_outage':{'status':200,'confirmed_message_preserved':True, + 'seconds_from_action_start':round(read_done-start,4), + 'seconds_from_stop_completion':round(read_done-stop_returned,4)}, + 'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']}) + finally: + cluster.command(['docker','start',current['container']]) + cluster.wait_healthy(current['container']) + cluster.wait_cluster() + time.sleep(2) + finally: + traffic.stop() + save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases, + 'total':traffic.summary(),'data':traffic.verify_data(), + 'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2}) + + +def rolling_update(): + cluster.wait_cluster() + before = cluster.replicas() + check(len(before)==2,'rolling demonstration expects two replicas') + lb_before = next(item for item in cluster.snapshot() if item['service']=='lb') + target_release = 'lab5-'+uuid.uuid4().hex[:8] + traffic = Traffic().start() + replacements = [] + try: + time.sleep(3) + build_start = time.monotonic() + cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180) + build_end = time.monotonic() + migration_output = cluster.compose('run','--rm','--no-deps','migrate') + swap_start = time.monotonic() + for old in before: + step_start = time.monotonic() + result = cluster.replace_one(old['container']) + result['seconds'] = round(time.monotonic()-step_start,4) + replacements.append(result) + swap_end = time.monotonic() + time.sleep(3) + finally: + traffic.stop() + after = cluster.replicas() + lb_after = next(item for item in cluster.snapshot() if item['service']=='lb') + check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test') + check(all(r['release']==target_release for r in after),'new release not observed on all replicas') + check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement') + check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change') + save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release, + 'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True, + 'build_seconds':round(build_end-build_start,4),'migration_output':migration_output, + 'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end), + 'total':traffic.summary(),'data':traffic.verify_data(), + 'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'}) + + +def database_fixture(container, data): + code = '''import json,sys,psycopg +from psycopg import sql +from messenger.db import setting +p=json.load(sys.stdin) +with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c: + if p['action']=='install': + c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text']))) + c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) + result={'installed':True} + elif p['action']=='remove': + c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger']))) + c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) + result={'removed':True} + else: + result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]} +print(json.dumps(result)) +''' + response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT, + input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(response.stdout) + + +def graceful_shutdown(): + cluster.wait_cluster() + nodes = cluster.replicas() + target,survivor = nodes[0],nodes[1] + address = cluster.front_address(target['container']) + c = Client() + alice,bob,_,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/messages' + marker = 'drain_'+uuid.uuid4().hex + fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker} + database_fixture(survivor['container'],{**fixture,'action':'install'}) + code = '''import json,sys,urllib.request,urllib.error +p=json.load(sys.stdin) +r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']}) +try: + response=urllib.request.urlopen(r,timeout=6) +except urllib.error.HTTPError as exc: + response=exc +print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)})) +''' + request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex} + def in_flight(): + result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT, + input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(result.stdout) + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(in_flight) + try: + deadline = time.monotonic()+3 + while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']: + check(time.monotonic()0 and item['nano_cpus']>0,'resource limit missing') + if item['service']!='lb': + check(not item['published_ports'],'service exposed a port bypassing LB') + check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas') + check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running') + save('infrastructure.json',{'result':'PASS','containers':state}) + + +def dependency_failures(): + c = Client(timeout=6) + actor = c.user('dependencies') + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result = {'result':'PASS'} + try: + cluster.compose('stop','-t','2','redis') + response = c.call('GET','/api/v1/users/me',actor=actor,schema='User') + check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed') + c.call('GET','/api/v1/users/me',401) + c.call('GET','/health/ready',schema='Health') + result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200} + finally: + cluster.compose('start','redis') + import urllib.request + import urllib.error + def status(path, token=None): + headers = {'Authorization':'Bearer '+token} if token else {} + request = urllib.request.Request(c.base+path,headers=headers) + started = time.monotonic() + try: + response = urllib.request.urlopen(request,timeout=6) + except urllib.error.HTTPError as error: + response = error + with response: + return response.code, round(time.monotonic()-started,4) + try: + cluster.compose('stop','-t','2','postgres') + code,elapsed = status('/api/v1/users/me',actor['token']) + check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request') + anonymous,_ = status('/api/v1/users/me') + check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage') + live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()] + check(all(code==200 for code in live),'API process liveness depends on DB') + result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous, + 'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'} + finally: + cluster.compose('start','postgres') + cluster.wait_cluster() + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result['same_session_after_recovery'] = True + save('dependency-failures.json',result) + + +def verify_nginx(): + state = cluster.snapshot() + lb = next(item for item in state if item['service']=='lb') + check(lb['image'].startswith('nginx:'),'load balancer must use Nginx') + cluster.command(['docker','exec',lb['container'],'nginx','-t']) + version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1']) + c = Client() + response = c.call('GET','/health/ready',schema='Health') + check(response['headers'].get('Server')=='nginx','public entry point is not Nginx') + check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas') + save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx', + 'config_syntax':'PASS','observed_instances':cluster.wait_cluster()}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?') + args = parser.parse_args() + modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover, + 'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure} + if args.mode=='all': + for function in modes.values(): + function() + else: + modes[args.mode]() + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_jobs.py b/scripts/verify_jobs.py new file mode 100644 index 00000000..0ce02bae --- /dev/null +++ b/scripts/verify_jobs.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Explicit real fault injection; emits IDs/results, never session tokens.""" +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + + +def compose(*args): + result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120) + if result.returncode: raise RuntimeError(result.stderr) + return result.stdout + + +def inside(body,**values): + script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()' + proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120) + if proc.returncode: raise RuntimeError(proc.stderr) + return json.loads(proc.stdout) + + +def state(aid): + return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid) + + +def wait_for(predicate,seconds=30): + end=time.monotonic()+seconds + while time.monotonic()0};obj.close()",aid=target['id']) + compose('kill','-s','SIGKILL','worker') + # unless-stopped can restart a killed worker; stop it to retain the fault boundary. + compose('stop','worker') + inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True") + trigger=False + crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt') + print('Waiting for real 90s lease expiry after killed worker.',flush=True) + started=time.monotonic() + wait_for(lambda: state(target['id'])['status']=='queued',110) + compose('start','worker') + ready(c,bob,target) + after=state(target['id']) + evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']} + # Re-deliver the completed original job. Ready metadata is immutable. + original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json'] + duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id']) + check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result') + evidence['duplicate']={'ready_noop':True} + # Retry exhaustion with real missing staging (transient storage failure), explicit replay. + compose('stop','worker','relay') + failed,failed_raw=c.attachment(alice,chat) + st=state(failed['id']) + inside("store.delete(v['key']);result=True",key=st['staging_key']) + for attempt in range(5): + inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id']) + failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json'] + check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id']) + compose('start','relay','worker') + ready(c,bob,failed) + evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409} + evidence['result']='PASS' + (ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n') + print('PASS: jobs-faults.json') + finally: + if trigger: + inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True") + compose('start','relay','worker') + + +if __name__=='__main__': main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..0156c23a --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,307 @@ +"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads.""" +import json +import asyncio +import logging +import os +import re +from contextlib import asynccontextmanager +from uuid import uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from psycopg_pool import PoolTimeout, TooManyRequests +from redis.exceptions import RedisError +from starlette.exceptions import HTTPException +from starlette.concurrency import run_in_threadpool + +from .db import Database +from .errors import ApiError, invalid, not_found, unauthorized +from .models import CreateChat, CreateMessage, CreateUser, uuid_value +from .messages import store_message, validate_key, MESSAGE_FIELDS +from .object_store import ObjectStore, unavailable +from . import attachments +from urllib.parse import quote +from .pagination import decode_cursor, page, page_limit, public, stamp +from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session + +LOG = logging.getLogger('messenger') +USER_FIELDS = 'id, username, display_name, created_at' + + +def error_response(request, error): + return JSONResponse({'error': {'code': error.code, 'message': error.message, + 'request_id': request.state.request_id}}, + status_code=error.status, headers=error.headers) + + +def authenticate(request: Request): + token = bearer(request.headers.get('Authorization', '')) + token_hash = digest(token) + with request.app.state.db.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', + (token_hash,)).fetchone() + if not row: + raise unauthorized() + request.state.token_hash = token_hash + return row['user_id'] + + +def require_member(conn, chat_id, user_id): + found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone() + if not found: + raise not_found() + + +def chat_public(conn, row): + item = public(row) + item['member_ids'] = [str(member['user_id']) for member in conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()] + return item + + +def create_app(): + @asynccontextmanager + async def lifespan(app): + app.state.db = Database() + app.state.storage = ObjectStore() + app.state.upload_slots = asyncio.Semaphore(6) + app.state.db.open() + try: + yield + finally: + app.state.storage.close() + app.state.db.close() + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + app.state.instance_id = uuid4().hex + app.state.release = os.environ.get('APP_RELEASE', 'lab5-v1') + + @app.middleware('http') + async def request_context(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path) + if request.method == 'POST' and json_endpoint: + media = request.headers.get('Content-Type', '').split(';') + unsupported = media[0].strip().lower() != 'application/json' + for parameter in media[1:]: + name, _, value = parameter.strip().partition('=') + if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'): + unsupported = True + if unsupported: + if path != '/api/v1/users': + await run_in_threadpool(authenticate, request) + raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8') + try: + json.loads((await request.body()).decode('utf-8')) + except (UnicodeError, json.JSONDecodeError): + raise invalid() from None + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc) + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests): + response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно')) + except Exception as exc: + # Log exception class only: credentials, SQL and message content stay out of logs. + LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__) + response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка')) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = app.state.instance_id + response.headers['X-App-Release'] = app.state.release + return response + + @app.exception_handler(ApiError) + async def api_error(request, exc): + return error_response(request, exc) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, exc): + return error_response(request, invalid()) + + @app.exception_handler(HTTPException) + async def http_error(request, exc): + return error_response(request, not_found() if exc.status_code == 404 else invalid()) + + @app.get('/health/live') + def live(): + return {'status': 'ok'} + + @app.get('/health/ready') + def ready(request: Request): + try: + with request.app.state.db.pool.connection() as conn: + conn.execute('SELECT 1 FROM idempotency_records LIMIT 0') + conn.execute('SELECT status,variants FROM attachments LIMIT 0') + conn.execute('SELECT 1 FROM job_outbox LIMIT 0') + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn): + raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None + request.app.state.storage.check() + return {'status': 'ok'} + + @app.post('/api/v1/users', status_code=201) + def register(body: CreateUser, request: Request): + password_hash = HASHER.hash(body.password) + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute( + f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}', + (uuid4(), body.username, body.display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Имя пользователя занято') from None + return public(row) + + @app.post('/api/v1/auth/sessions', status_code=201) + async def create_session(request: Request): + # Run hashing and synchronous database calls outside the event loop. + username, password = basic_credentials(request.headers.get('Authorization', '')) + if await request.body(): + raise invalid() + return await run_in_threadpool(login, request, username, password) + + def login(request, username, password): + db = request.app.state.db + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone() + valid = check_password(row['password_hash'] if row else DUMMY_HASH, password) + if not row or not valid: + raise unauthorized(basic=True) + row.pop('password_hash') + token, expires_at = new_session(db.session_ttl) + with db.pool.connection() as conn: + conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)', + (digest(token), row['id'], expires_at)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)} + + @app.get('/api/v1/users/me') + def me(request: Request, user_id=Depends(authenticate)): + db = request.app.state.db + key = 'user:' + str(user_id) + try: + cached = db.cache.get(key) + if cached: + return json.loads(cached) + except (RedisError, ValueError): + pass + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone() + if not row: + raise unauthorized() + result = public(row) + try: + db.cache.set(key, json.dumps(result), ex=60) + except RedisError: + pass + return result + + @app.delete('/api/v1/auth/sessions/current', status_code=204) + def logout(request: Request, user_id=Depends(authenticate)): + with request.app.state.db.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,)) + return Response(status_code=204) + + @app.post('/api/v1/chats', status_code=201) + def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)): + if str(user_id) not in body.member_ids: + raise invalid() + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at', + (uuid4(), body.title)).fetchone() + # An unknown user raises AFTER the chat INSERT; transaction context rolls it all back. + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', + [(row['id'], uuid_value(member)) for member in body.member_ids]) + result = chat_public(conn, row) + except psycopg.errors.ForeignKeyViolation: + raise invalid() from None + return result + + @app.get('/api/v1/chats') + def list_chats(request: Request, user_id=Depends(authenticate)): + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'chats:' + str(user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (c.created_at,c.id) > (%s,%s)' if after else '' + params = (user_id, *after, limit+1) if after else (user_id, limit+1) + with request.app.state.db.pool.connection() as conn: + rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall() + result = page(scope, rows, limit) + result['items'] = [chat_public(conn, row) for row in rows[:limit]] + return result + + @app.get('/api/v1/chats/{chat_id}') + def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone() + return chat_public(conn, row) + + @app.post('/api/v1/chats/{chat_id}/messages', status_code=201) + def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + key = validate_key(request.headers.get('Idempotency-Key', '')) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids) + return public(row) + + @app.get('/api/v1/chats/{chat_id}/messages') + def history(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'messages:' + str(cid) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (m.created_at,m.id) > (%s,%s)' if after else '' + params = (cid, *after, limit+1) if after else (cid, limit+1) + rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+ + ' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall() + return page(scope, rows, limit) + + @app.post('/api/v1/chats/{chat_id}/attachments',status_code=202) + async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)): + cid=uuid_value(chat_id) + def access(): + with request.app.state.db.pool.connection() as conn: + require_member(conn,cid,user_id) + await run_in_threadpool(access) + return await attachments.upload(request,user_id,cid,filename) + + @app.get('/api/v1/attachments/{attachment_id}') + def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + return attachments.metadata(row) + + @app.get('/api/v1/attachments/{attachment_id}/content') + def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)): + aid=uuid_value(attachment_id) + with request.app.state.db.pool.connection() as conn: + row=attachments.lookup(conn,aid,user_id) + if variant not in ('original','thumbnail','crop','optimized'): + raise invalid() + if row['status']!='ready': + raise ApiError(409,'conflict','Вложение ещё не готово') + item=row + key=row['object_key'] + if variant!='original': + item=next((v for v in row['variants'] if v['name']==variant),None) + if item is None: + raise not_found() + key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant + opened=request.app.state.storage.get(key) + if opened.length!=item['size_bytes']: + opened.close() + raise unavailable() + return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={ + 'Content-Length':str(item['size_bytes']), + 'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')}) + + return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..5e7eadb8 --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,173 @@ +"""Bounded binary upload, image validation and DB/S3 coordination.""" +import asyncio +import hashlib +import logging +import tempfile +import threading +import unicodedata +import warnings +from uuid import uuid4 + +import psycopg +from PIL import Image, PngImagePlugin, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from .errors import ApiError, invalid, not_found +from .pagination import public + +MAX_BYTES=10485760 +MAX_PIXELS=40000000 +Image.MAX_IMAGE_PIXELS=MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK=1048576 +PngImagePlugin.MAX_TEXT_MEMORY=1048576 +warnings.filterwarnings('error',category=Image.DecompressionBombWarning) +DECODER=threading.BoundedSemaphore(1) +LOG=logging.getLogger('messenger') +FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants' + + +def object_lock(conn,key,wait=True): + function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' + row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone() + return True if wait else row['locked'] + + +def filename_value(value): + if not value or not value.strip() or len(value)>255: + raise invalid() + try: + value.encode('utf-8') + except UnicodeError: + raise invalid() from None + if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value): + raise invalid() + return value + + +def metadata(row): + result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')}) + return result + + +def validate_image(file,mime): + if not DECODER.acquire(timeout=3): + raise ApiError(503,'unavailable','Обработка изображений занята') + try: + file.seek(0) + with Image.open(file) as image: + actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format) + if actual is None or actual!=mime: + raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME') + if image.width*image.height>MAX_PIXELS: + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') + image.verify() + file.seek(0) + except (Image.DecompressionBombWarning,Image.DecompressionBombError): + raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None + except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError): + raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None + finally: + DECODER.release() + + +def compensate(db,store,key): + """A failed/unknown COMMIT is not proof of absence. Recheck under the same lock.""" + try: + with db.pool.connection() as conn: + if not object_lock(conn,key,wait=False): + return + exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone() + if not exists: + store.delete(key) + except Exception as exc: + # A periodic, grace-protected janitor will retry after dependency recovery. + LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__) + + +def persist(db,store,user_id,chat_id,filename,mime,file,length,digest): + aid=uuid4() + key='staging/'+aid.hex + try: + with db.pool.connection() as conn: + object_lock(conn,key) + store.put(key,file,length,mime) + row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) ' + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS, + (aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone() + job=uuid4() + conn.execute('INSERT INTO attachment_jobs(id,attachment_id) VALUES (%s,%s)',(job,aid)) + conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,)) + return metadata(row) + except psycopg.Error: + compensate(db,store,key) + raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None + except BaseException: + compensate(db,store,key) + raise + + +async def upload(request,user_id,chat_id,filename): + filename=filename_value(filename) + mime=request.headers.get('Content-Type','').split(';')[0].strip().lower() + if mime not in ('image/png','image/jpeg'): + raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg') + declared=request.headers.get('Content-Length') + if declared is not None: + try: + length=int(declared) + if length<0: + raise ValueError + except ValueError: + raise invalid() from None + if length>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + try: + await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1) + except TimeoutError: + raise ApiError(503,'unavailable','Слишком много загрузок') from None + try: + with tempfile.TemporaryFile(mode='w+b') as file: + total=0 + digest=hashlib.sha256() + async def read_body(): + nonlocal total + async for chunk in request.stream(): + total+=len(chunk) + if total>MAX_BYTES: + raise ApiError(413,'payload_too_large','Файл превышает 10 MiB') + digest.update(chunk) + await run_in_threadpool(file.write,chunk) + try: + await asyncio.wait_for(read_body(),timeout=30) + except TimeoutError: + raise ApiError(400,'invalid_request','Время загрузки истекло') from None + except ClientDisconnect: + raise invalid() from None + await run_in_threadpool(validate_image,file,mime) + return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage, + user_id,chat_id,filename,mime,file,total,digest.hexdigest()) + finally: + request.app.state.upload_slots.release() + + +def lookup(conn,attachment_id,user_id): + row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+ + ' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone() + if not row: + raise not_found() + return row + + +class ObjectResponse(StreamingResponse): + def __init__(self,opened,**kwargs): + self.opened=opened + super().__init__(opened.chunks(),**kwargs) + + async def __call__(self,scope,receive,send): + try: + await super().__call__(scope,receive,send) + finally: + self.opened.close() diff --git a/src/messenger/cleanup.py b/src/messenger/cleanup.py new file mode 100644 index 00000000..d5b5dfab --- /dev/null +++ b/src/messenger/cleanup.py @@ -0,0 +1,72 @@ +"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock.""" +import argparse +from datetime import datetime,timezone,timedelta +import json +from pathlib import Path +import time + +from .attachments import object_lock +from .db import Database +from .object_store import ObjectStore + + +def sweep(db,store,grace_seconds=300,apply=False): + if grace_seconds<60: + raise ValueError('Grace must be at least60seconds') + cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds) + result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds, + 'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0} + for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)): + for obj in page.get('Contents',[]): + result['listed']+=1 + if obj['LastModified']>=cutoff: + result['fresh']+=1 + continue + with db.pool.connection() as conn: + if not object_lock(conn,obj['Key'],wait=False): + result['locked']+=1 + continue + if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone(): + result['tracked']+=1 + continue + result['orphans']+=1 + if apply: + store.delete(obj['Key']) + result['deleted']+=1 + return result + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--grace-seconds',type=int,default=300) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--dry-run',action='store_true') + parser.add_argument('--loop',action='store_true') + parser.add_argument('--interval',type=int,default=60) + args=parser.parse_args() + if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run): + parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run') + db=Database() + store=ObjectStore() + db.open() + try: + while True: + try: + result=sweep(db,store,args.grace_seconds,args.apply) + print(json.dumps(result),flush=True) + except Exception as exc: + print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True) + if not args.loop: + raise + finally: + Path('/tmp/cleanup-heartbeat').touch() + if not args.loop: + break + time.sleep(args.interval) + finally: + store.close() + db.close() + + +if __name__=='__main__': + main() diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..27356ec1 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,42 @@ +"""Bounded dependencies; PostgreSQL is always the source of truth.""" +import os +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +def setting(name, default=None): + filename = os.environ.get(name + '_FILE') + value = Path(filename).read_text().strip() if filename else os.environ.get(name, default) + if value is None: + raise RuntimeError(f'Missing configuration: {name}') + return value + + +class Database: + def __init__(self): + self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600')) + if self.session_ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + self.pool = ConnectionPool( + setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')), + timeout=2, max_waiting=16, open=False, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'}, + ) + self.cache = Redis.from_url( + setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8, + retry=Retry(NoBackoff(), 0), + ) + + def open(self): + self.pool.open(wait=True, timeout=30) + + def close(self): + self.pool.close() + self.cache.close() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..082390e7 --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,19 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(): + return ApiError(400, 'invalid_request', 'Неверный запрос') + + +def unauthorized(basic=False): + scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme}) + + +def not_found(): + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/jobs.py b/src/messenger/jobs.py new file mode 100644 index 00000000..79f19a9d --- /dev/null +++ b/src/messenger/jobs.py @@ -0,0 +1,222 @@ +"""Transactional outbox, fenced processing, bounded retries and operator replay.""" +from contextlib import nullcontext +import signal +import hashlib +import json +import logging +import os +from pathlib import Path +import random +import subprocess +import sys +import tempfile +from uuid import UUID + +from psycopg.types.json import Jsonb +from redis import Redis +from .db import setting +from .attachments import object_lock + +LOG=logging.getLogger('messenger') +STREAM='attachment-jobs' +GROUP='image-workers' +MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5')) +LEASE_SECONDS=90 + + +def broker_client(): + return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True, + socket_connect_timeout=1,socket_timeout=3,max_connections=4) + + +def job_lock(conn,job_id): + return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok', + ('job:'+str(job_id),)).fetchone()['ok'] + + +def unlock(conn,job_id): + conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),)) + + +def release_lock(conn,job_id): + # Session locks survive ROLLBACK. A failed transaction must be cleared before unlock. + try: + conn.rollback() + unlock(conn,job_id) + conn.commit() + except BaseException: + conn.close() # never return a session with an uncertain held lock to the pool + raise + + +def relay_tick(db,broker): + # Recovery uses the same session lock as the processor. A slow live worker is never fenced out. + with db.pool.connection() as conn: + expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE a.status='processing' AND j.lease_untilrow['size_bytes'] or total>10485760: raise ValueError('invalid_image') + file.write(chunk) + finally: + opened.close() + raw_hash=hashlib.sha256(source.read_bytes()).hexdigest() + if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']: + raise ValueError('invalid_image') + completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory], + capture_output=True,timeout=20) + if completed.returncode: + raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit') + variants=json.loads(completed.stdout) + with source.open('rb') as file: + store.put(row['object_key'],file,row['size_bytes'],row['content_type']) + for name,item in variants.items(): + with (Path(directory)/name).open('rb') as file: + store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type']) + return list(variants.values()) + + +def process_objects(store,row): + # A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread. + timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45')) + if not 1<=timeout<=45: raise ValueError('processing_limit') + with tempfile.TemporaryDirectory(prefix='image-job-') as directory: + process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory], + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE, + text=True,start_new_session=True) + try: + output,_=process.communicate(json.dumps(row,default=str),timeout=timeout) + except BaseException: + # Terminate processor AND decoder before lock release or another attempt. + try: os.killpg(process.pid,signal.SIGKILL) + except ProcessLookupError: pass + process.communicate() + raise + if process.returncode==65: raise ValueError('invalid_image') + if process.returncode==66: raise ValueError('processing_limit') + if process.returncode: raise RuntimeError('storage_unavailable') + return json.loads(output) + + +def work(db,store,job_id,delivery_attempt=None): + job_id=UUID(str(job_id)) + # Dedicated pooled session holds the lock, with short committed transactions between effects. + with db.pool.connection() as conn: + if not job_lock(conn,job_id): + return False + conn.commit() + try: + row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due ' + 'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone() + conn.commit() + if not row or row['status'] in ('ready','failed'): + return True + if row['status']!='queued' or not row['due']: + return False + if delivery_attempt is not None and delivery_attempt!=row['attempt']+1: + return True + attempt=row['attempt']+1 + if attempt>MAX_ATTEMPTS: + conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.commit() + return True + conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s", + (attempt,LEASE_SECONDS,job_id)) + conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.commit() + try: + variants=process_objects(store,row) + except Exception as exc: + permanent=isinstance(exc,ValueError) + final=permanent or attempt>=MAX_ATTEMPTS + code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable') + if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'): + code='processing_failed' + delay=min(30,2**(attempt-1))*random.uniform(.5,1.5) + conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + ('failed' if final else 'queued',code if final else None,row['id'])) + conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id)) + if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,)) + conn.commit() + LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt) + return True + # Same live lock and attempt fence protect publication; no new attempt can publish concurrently. + current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone() + if current['attempt']!=attempt: + conn.rollback() + return False + conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (Jsonb(variants),row['id'])) + conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,)) + conn.commit() + LOG.info('job_completed attempt=%s',attempt) + clean_staging(db,store,row['id']) + return True + finally: + release_lock(conn,job_id) + + +def clean_staging(db,store,attachment_id=None): + with db.pool.connection() as conn: + rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL " + "AND (status='ready' OR (status='failed' AND updated_at200: raise ValueError('Short audit reason required') + with db.pool.connection() as conn: + if not job_lock(conn,job_id): raise ValueError('Job active') + try: + row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone() + if not row or row['status']!='failed' or not row['staging_key']: + raise ValueError('Replay requires failed job with retained staging') + conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,)) + conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,)) + conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason)) + conn.commit() + finally: + release_lock(conn,job_id) diff --git a/src/messenger/messages.py b/src/messenger/messages.py new file mode 100644 index 00000000..06599672 --- /dev/null +++ b/src/messenger/messages.py @@ -0,0 +1,54 @@ +"""One PostgreSQL transaction owns both a message and its retry result.""" +import hashlib +import json +import re +from uuid import uuid4 + +from .errors import ApiError, invalid, not_found +from .models import uuid_value + + +def validate_key(key): + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise invalid() + return key + + +MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids' + + +def store_message(conn, user_id, chat_id, text, key, attachment_ids=None): + attachment_ids=[] if attachment_ids is None else attachment_ids + payload={'text':text} + if attachment_ids: + payload['attachment_ids']=attachment_ids + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + params = (user_id, chat_id, key) + # A competing INSERT waits for the other transaction, then the next statement sees its result. + conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING", + (*params, fingerprint)) + record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active ' + 'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone() + if record['message_id'] is not None and record['active']: + if record['request_hash'] != fingerprint: + raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением') + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s', + (record['message_id'],)).fetchone() + ids=[uuid_value(item) for item in attachment_ids] + if ids: + allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall() + if len(allowed)!=len(ids): + raise not_found() + if any(a['status']=='failed' for a in allowed): + raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой') + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone() + if ids: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)', + [(row['id'],aid,i) for i,aid in enumerate(ids)]) + conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' " + 'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params)) + return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e591e4ee --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,29 @@ +"""Versioned, atomic, repeatable migrations, serialized with an advisory lock.""" +import hashlib +from pathlib import Path + +import psycopg +from .db import setting + + +def main(): + directory = Path(__file__).resolve().parents[2] / 'migrations' + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(20261102)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(directory.glob('*.sql')): + sql = path.read_text() + checksum = hashlib.sha256(sql.encode()).hexdigest() + found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone() + if found: + if found[0] != checksum: + raise RuntimeError('Applied migration checksum changed: ' + path.name) + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum)) + print('Applied', path.name) + print('Migrations up to date') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..85e68919 --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,75 @@ +import re +from uuid import UUID +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from .errors import invalid + + +def uuid_value(value): + try: + # Require the UUID string representation, not integer/coercible values. + if not isinstance(value, str) or len(value) != 36: + raise ValueError + return UUID(value) + except (ValueError, TypeError, AttributeError): + raise invalid() from None + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + @field_validator('*') + @classmethod + def valid_strings(cls, value, info): + if isinstance(value, str): + if not value.strip() and info.field_name != 'text': + raise ValueError('Blank string') + # PostgreSQL cannot store NUL or unpaired Unicode surrogates. + if '\x00' in value: + raise ValueError('NUL is not supported') + value.encode('utf-8') + return value + + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('username') + @classmethod + def username_ascii(cls, value): + if not re.fullmatch(r'[a-z0-9_]{3,32}', value): + raise ValueError('Invalid username') + return value + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('member_ids') + @classmethod + def members(cls, value): + ids = [uuid_value(item) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('Duplicate member') + return [str(item) for item in ids] + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list,max_length=10) + + @field_validator('attachment_ids') + @classmethod + def attachment_values(cls,value): + ids=[uuid_value(item) for item in value] + if len(ids)!=len(set(ids)): + raise ValueError('Duplicate attachment') + return [str(item) for item in ids] + + @model_validator(mode='after') + def nonempty_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('Message must contain text or attachment') + return self diff --git a/src/messenger/object_store.py b/src/messenger/object_store.py new file mode 100644 index 00000000..c778bcc0 --- /dev/null +++ b/src/messenger/object_store.py @@ -0,0 +1,109 @@ +"""Small bounded S3 client. API credentials never contain root/admin permissions.""" +from contextlib import contextmanager +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from .db import setting +from .errors import ApiError + +PREFIX = 'originals/' + + +def unavailable(): + return ApiError(503,'unavailable','Хранилище файлов временно недоступно') + + +class OpenObject: + def __init__(self,body,release,length): + self.body=body + self.release=release + self.length=length + self.closed=False + self.close_lock=threading.Lock() + + def chunks(self): + try: + while True: + data=self.body.read(65536) + if not data: + break + yield data + finally: + self.close() + + def close(self): + with self.close_lock: + if self.closed: + return + self.closed=True + try: + self.body.close() + finally: + self.release() + + +class ObjectStore: + def __init__(self): + self.bucket=setting('S3_BUCKET','messenger-attachments') + self.gate=threading.BoundedSemaphore(8) + self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'), + region_name=setting('S3_REGION','us-east-1'), + aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), + config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8, + retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'}, + request_checksum_calculation='when_required',response_checksum_validation='when_required')) + + @contextmanager + def operation(self): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + yield + except (BotoCoreError,ClientError): + raise unavailable() from None + finally: + self.gate.release() + + def check(self): + with self.operation(): + self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1) + + def put(self,key,body,length,mime): + with self.operation(): + body.seek(0) + self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime) + + def delete(self,key): + with self.operation(): + self.client.delete_object(Bucket=self.bucket,Key=key) + + def get(self,key): + if not self.gate.acquire(timeout=1): + raise unavailable() + try: + result=self.client.get_object(Bucket=self.bucket,Key=key) + return OpenObject(result['Body'],self.gate.release,result['ContentLength']) + except (BotoCoreError,ClientError): + self.gate.release() + raise unavailable() from None + except BaseException: + self.gate.release() + raise + + def pages(self,prefix=PREFIX): + token=None + while True: + params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000} + if token: + params['ContinuationToken']=token + with self.operation(): + result=self.client.list_objects_v2(**params) + yield result + if not result.get('IsTruncated'): + break + token=result['NextContinuationToken'] + + def close(self): + self.client.close() diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..14564c58 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,56 @@ +import base64 +import json +import re +from datetime import datetime, timezone +from uuid import UUID +from .errors import invalid +from .models import uuid_value + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def encode_cursor(scope, row): + raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip('=') + + +def decode_cursor(scope, value): + if value is None: + return None + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value): + raise ValueError + data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + raise ValueError + ts = datetime.fromisoformat(data[1].replace('Z', '+00:00')) + if ts.tzinfo is None: + raise ValueError + return ts, uuid_value(data[2]) + except (ValueError, TypeError, AttributeError, OverflowError): + raise invalid() from None + + +def page_limit(value): + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + raise invalid() + return int(value) + + +def public(row): + def value(item): + if isinstance(item,datetime): + return stamp(item) + if isinstance(item,UUID): + return str(item) + if isinstance(item,list): + return [value(part) for part in item] + return item + return {key:value(item) for key,item in row.items()} + + +def page(scope, rows, limit): + return {'items': [public(row) for row in rows[:limit]], + 'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None} diff --git a/src/messenger/pipeline.py b/src/messenger/pipeline.py new file mode 100644 index 00000000..806b4316 --- /dev/null +++ b/src/messenger/pipeline.py @@ -0,0 +1,49 @@ +"""Bounded image decoder subprocess. Original bytes are never re-encoded.""" +import hashlib +import json +import resource +import sys +from pathlib import Path +from PIL import Image, ImageOps, PngImagePlugin + + +def fitted(image, maximum): + scale=min(1,maximum/image.width,maximum/image.height) + size=(max(1,int(image.width*scale)),max(1,int(image.height*scale))) + return image.resize(size,Image.Resampling.LANCZOS) + + +def process_file(source, directory): + Image.MAX_IMAGE_PIXELS=40000000 + PngImagePlugin.MAX_TEXT_CHUNK=1048576 + PngImagePlugin.MAX_TEXT_MEMORY=1048576 + with Image.open(source) as original: + if original.format not in ('PNG','JPEG') or original.width*original.height>40000000: + raise ValueError('invalid_image') + image=ImageOps.exif_transpose(original) + # Alpha is composited on white; output carries no source metadata. + rgba=image.convert('RGBA') + white=Image.new('RGBA',rgba.size,'white') + white.alpha_composite(rgba) + image=white.convert('RGB') + image.info.clear() + outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'), + 'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'), + 'optimized':(fitted(image,1280),'WEBP','image/webp')} + manifest={} + for name,(variant,fmt,mime) in outputs.items(): + path=Path(directory)/name + variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'') + raw=path.read_bytes() + manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height, + 'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()} + return manifest + + +if __name__=='__main__': + resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024)) + resource.setrlimit(resource.RLIMIT_CPU,(15,15)) + try: + print(json.dumps(process_file(sys.argv[1],sys.argv[2]))) + except (OSError,ValueError,MemoryError,Image.DecompressionBombError): + sys.exit(65) diff --git a/src/messenger/processor.py b/src/messenger/processor.py new file mode 100644 index 00000000..6753dfd4 --- /dev/null +++ b/src/messenger/processor.py @@ -0,0 +1,29 @@ +"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds.""" +import os +import signal +import json +import sys +from uuid import UUID +from .jobs import process_objects_local +from .object_store import ObjectStore + + +def main(): + # Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline. + signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL)) + signal.alarm(45) + row=json.load(sys.stdin) + row['id']=UUID(row['id']) + store=ObjectStore() + try: + print(json.dumps(process_objects_local(store,row,sys.argv[1]))) + except ValueError as exc: + sys.exit(66 if str(exc)=='processing_limit' else 65) + except Exception: + # Do not export credentials, storage keys or SDK response details. + sys.exit(75) + finally: + store.close() + + +if __name__=='__main__': main() diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..b82c6b0c --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,49 @@ +import base64 +import hashlib +import re +import secrets +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from .errors import unauthorized + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash('synthetic-dummy-password') + + +def digest(token): + return hashlib.sha256(token.encode()).hexdigest() + + +def basic_credentials(header): + try: + scheme, encoded = header.split(' ', 1) + if scheme.lower() != 'basic': + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError): + raise unauthorized(basic=True) from None + + +def check_password(encoded, password): + try: + return HASHER.verify(encoded, password) + except (VerificationError, InvalidHashError): + return False + + +def bearer(header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError + return token + except ValueError: + raise unauthorized() from None + + +def new_session(ttl): + return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl) diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 00000000..19b60676 --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,69 @@ +"""Separate relay/consumer processes; durable Redis Streams consumer group.""" +import argparse +import json +import logging +import signal +import time +from uuid import uuid4 +from redis.exceptions import ResponseError +from .cleanup import sweep +from .db import Database +from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP +from .object_store import ObjectStore + +LOG=logging.getLogger('messenger') +STOP=False + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('role',choices=['worker','relay','replay']) + parser.add_argument('--job-id') + parser.add_argument('--reason',default='operator retry') + args=parser.parse_args() + logging.basicConfig(level=logging.INFO) + def stop(*_): + global STOP + STOP=True + signal.signal(signal.SIGTERM,stop) + signal.signal(signal.SIGINT,stop) + db=Database();db.open();store=ObjectStore();broker=broker_client() + consumer=uuid4().hex + last_cleanup=0 + try: + if args.role=='replay': + replay(db,args.job_id,args.reason) + print('Replay queued with audit') + return + if args.role=='worker': + try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True) + except ResponseError as exc: + if 'BUSYGROUP' not in str(exc): raise + while not STOP: + try: + if args.role=='relay': + relay_tick(db,broker) + clean_staging(db,store) + if time.monotonic()-last_cleanup>60: + sweep(db,store,300,True) + last_cleanup=time.monotonic() + time.sleep(.3) + else: + claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=90000,start_id='0-0',count=1) + messages=claimed[1] + if not messages: + batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000) + messages=batches[0][1] if batches else [] + for message_id,fields in messages: + payload=json.loads(fields['payload']) + if work(db,store,payload['job_id'],payload['attempt']): + broker.xack(STREAM,GROUP,message_id) + broker.xdel(STREAM,message_id) + except Exception as exc: + LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__) + time.sleep(1) + finally: + broker.close();store.close();db.close() + + +if __name__=='__main__': main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..6342a6ae --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,141 @@ +"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities.""" +import base64 +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fastapi.testclient import TestClient +from messenger.app import create_app + +PASSWORD = 'Тестовый:пароль🔐' * 8 + + +@pytest.fixture +def client(): + with TestClient(create_app()) as c: + yield c + + +def user(c): + body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD) + r = c.post('/api/v1/users', json=body) + assert r.status_code == 201, r.text + return r.json(), body + + +def login(c, body): + credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode() + r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials}) + assert r.status_code == 201, r.text + return {'Authorization': 'Bearer ' + r.json()['token']}, r.json() + + +def test_auth_unicode_logout_and_cache(client): + u, body = user(client) + h, session = login(client, body) + assert session['user'] == u + assert len(session['token']) >= 43 + key = 'user:' + u['id'] + client.app.state.db.cache.delete(key) + for _ in range(2): + assert client.get('/api/v1/users/me', headers=h).json() == u + cached = json.loads(client.app.state.db.cache.get(key)) + assert cached == u + assert 0 < client.app.state.db.cache.ttl(key) <= 60 + assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204 + r = client.get('/api/v1/users/me', headers=h) + assert r.status_code == 401 + assert r.headers['www-authenticate'] == 'Bearer realm="messenger"' + assert r.json()['error']['request_id'] == r.headers['x-request-id'] + + +def test_session_expiry(monkeypatch): + monkeypatch.setenv('SESSION_TTL_SECONDS', '2') + with TestClient(create_app()) as client: + _, body = user(client) + h, _ = login(client, body) + assert client.get('/api/v1/users/me', headers=h).status_code == 200 + time.sleep(2.1) + assert client.get('/api/v1/users/me', headers=h).status_code == 401 + + +def test_registration_race(client): + body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD) + with ThreadPoolExecutor(max_workers=6) as executor: + codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6))) + assert sorted(codes) == [201, 409, 409, 409, 409, 409] + + +def test_chat_rollback_after_insert(client): + u, body = user(client) + h, _ = login(client, body) + title = 'rollback_' + uuid.uuid4().hex + r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]}) + assert r.status_code == 400 + with client.app.state.db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0 + + +def test_cursor_access_and_strict_validation(client): + a, ab = user(client) + b, bb = user(client) + ah, _ = login(client, ab) + bh, _ = login(client, bb) + chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)] + path = '/api/v1/chats/' + chats[0]['id'] + for text in ['Один 👋', 'Два', 'Три']: + assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201 + p = client.get(path+'/messages?limit=2', headers=ah).json() + assert [m['text'] for m in p['items']] == ['Один 👋','Два'] + tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json() + assert [m['text'] for m in tail['items']] == ['Три'] + assert tail['next_cursor'] is None + r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']}) + assert r.status_code == 400 + assert client.get(path, headers=bh).status_code == 404 + for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]: + assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400 + for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']: + assert client.get(path+'/messages?'+query,headers=ah).status_code == 400 + for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]: + assert client.post('/api/v1/users',json=payload).status_code == 400 + assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400 + + +def test_session_storage_and_migrations(client): + from messenger.migrate import main as migrate + u, body = user(client) + _, session = login(client, body) + db = client.app.state.db + def snapshot(): + with db.pool.connection() as conn: + return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] + for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']} + before = snapshot() + migrate() + migrate() + assert snapshot() == before + with db.pool.connection() as conn: + row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone() + assert row['token_hash'] != session['token'] + assert len(row['token_hash']) == 64 + stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash'] + assert stored.startswith('$argon2id$') + assert stored != body['password'] + + +def test_json_media_type_and_utf8(client): + body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD) + raw = json.dumps(body).encode() + for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]: + r = client.post('/api/v1/users',content=raw,headers=headers) + assert r.status_code == 415 + assert r.json()['error']['code'] == 'unsupported_media_type' + for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']: + r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'}) + assert r.status_code == 400, encoding + r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'}) + assert r.status_code == 401 diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..a5b1dfad --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,187 @@ +"""Real PostgreSQL + private S3 integration; synthetic images only.""" +import hashlib +import io +import struct +import uuid +import zlib +from PIL import Image +from psycopg import sql +from test_api import client, user, login + + +def image_bytes(kind='PNG'): + stream = io.BytesIO() + Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind) + return stream.getvalue() + + +def actors(c): + a,ab=user(c) + b,bb=user(c) + e,eb=user(c) + ah,_=login(c,ab) + bh,_=login(c,bb) + eh,_=login(c,eb) + chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json() + return ah,bh,eh,chat + + +def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'): + raw=image_bytes() if raw is None else raw + return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw, + headers={**headers,'Content-Type':mime}) + + +def finish(client,attachment_id): + from messenger.jobs import work + with client.app.state.db.pool.connection() as conn: + job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id'] + work(client.app.state.db,client.app.state.storage,job) + import time + for _ in range(100): + with client.app.state.db.pool.connection() as conn: + status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status'] + if status=='ready': return + time.sleep(.1) + raise AssertionError('Job did not become ready: '+status) + + +def test_original_png_jpeg_and_message(client): + ah,bh,eh,chat=actors(client) + for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]: + raw=image_bytes(kind) + r=upload(client,ah,chat,raw,mime) + assert r.status_code==202 + att=r.json() + assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None + assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409 + finish(client,att['id']) + att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json() + assert att['status']=='ready' and len(att['variants'])==3 + assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw) + assert 'object_key' not in att and 'bucket' not in att + path='/api/v1/attachments/'+att['id'] + assert client.get(path,headers=bh).json()==att + content=client.get(path+'/content',headers=bh) + assert content.status_code==200 and content.content==raw + assert content.headers['Content-Type']==mime + for suffix in ['', '/content']: + assert client.get(path+suffix,headers=eh).status_code==404 + assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200 + response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, + json={'text':'','attachment_ids':[att['id']]}) + assert response.status_code==201 + assert response.json()['attachment_ids']==[att['id']] + + +def test_invalid_media_corruption_limits_and_filename(client): + ah,_,_,chat=actors(client) + cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415), + (image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415), + (image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)] + for raw,mime,expected in cases: + response=upload(client,ah,chat,raw,mime) + assert response.status_code==expected + assert response.json()['error']['request_id']==response.headers['X-Request-Id'] + # Valid PNG header advertises >40Mp; reject before attempting decompression. + header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0) + chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff) + bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'') + assert upload(client,ah,chat,bomb).status_code==413 + for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]: + assert upload(client,ah,chat,filename=filename).status_code==400 + + +def test_foreign_attachment_binding_and_normalized_replay(client): + ah,bh,eh,chat=actors(client) + att=upload(client,ah,chat).json() + other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json() + path='/api/v1/chats/'+other['id']+'/messages' + for aid in [att['id'],str(uuid.uuid4())]: + response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]}) + assert response.status_code==404 + assert upload(client,eh,chat).status_code==404 + h={**ah,'Idempotency-Key':uuid.uuid4().hex} + path='/api/v1/chats/'+chat['id']+'/messages' + first=client.post(path,headers=h,json={'text':'same'}) + replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'}) + assert first.status_code==replay.status_code==201 and first.json()==replay.json() + conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]}) + assert conflict.status_code==409 + duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]}) + assert duplicates.status_code==400 + + +def test_compensate_s3_when_metadata_insert_fails(client): + ah,_,_,chat=actors(client) + db=client.app.state.db + store=client.app.state.storage + before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])} + function='fail_attachment_'+uuid.uuid4().hex + trigger='fail_attachment_'+uuid.uuid4().hex + filename='fault_'+uuid.uuid4().hex+'.png' + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename))) + conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + response=upload(client,ah,chat,filename=filename) + assert response.status_code==503 + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0 + after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])} + assert after==before + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + + +def test_compensation_preserves_committed_object(client): + from messenger.attachments import compensate + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + # A retry after an unknown COMMIT result must never delete an already tracked object. + compensate(client.app.state.db,client.app.state.storage,key) + content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah) + assert content.status_code==200 and content.content==image_bytes() + + +def test_stream_close_even_if_headers_cannot_be_sent(client): + import asyncio + import pytest + from messenger.attachments import ObjectResponse + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + with client.app.state.db.pool.connection() as conn: + key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key'] + store=client.app.state.storage + opened=store.get(key) + response=ObjectResponse(opened,media_type='image/png') + async def fail_send(message): + raise OSError('synthetic disconnected transport') + async def receive(): + return {'type':'http.disconnect'} + with pytest.raises(Exception): + asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send)) + assert opened.closed + acquired=0 + try: + for _ in range(8): + assert store.gate.acquire(blocking=False) + acquired+=1 + finally: + for _ in range(acquired): + store.gate.release() + + +def test_chunked_upload_is_limited_without_content_length(client): + ah,_,_,chat=actors(client) + chunks=iter([b'x'*1048576]*10+[b'x']) + response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', + content=chunks,headers={**ah,'Content-Type':'image/png'}) + assert response.status_code==413 + assert response.json()['error']['code']=='payload_too_large' diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..826d3dc1 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,119 @@ +"""Regressions for lab3 shared message results, real PostgreSQL (no mocks).""" +from concurrent.futures import ThreadPoolExecutor +import uuid + +from psycopg import sql +from test_api import client, user, login + + +def setup_chat(c, extra=None): + u, body = user(c) + headers, _ = login(c, body) + members = [u['id']] + (extra or []) + chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json() + return u, headers, '/api/v1/chats/'+chat['id']+'/messages' + + +def test_instance_alias_on_success_and_errors(client): + responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'), + client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})] + ids = [r.headers.get('X-Instance-Id') for r in responses] + assert ids[0] and len(set(ids)) == 1 + assert all(r.headers.get('X-Request-Id') for r in responses) + + +def test_key_required_replay_and_conflict(client): + _, headers, path = setup_chat(client) + assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400 + for key in ['', 'a b', 'bad/', 'x'*129]: + assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400 + h = {**headers, 'Idempotency-Key':'valid:._-123'} + first = client.post(path, headers=h, json={'text':'same'}) + assert first.status_code == 201 + replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ') + assert replay.status_code == 201 and replay.json() == first.json() + assert client.post(path, headers=h, json={'text':'other'}).status_code == 409 + assert len(client.get(path,headers=headers).json()['items']) == 1 + + +def test_concurrent_key_produces_one_message(client): + _, headers, path = setup_chat(client) + h = {**headers, 'Idempotency-Key':uuid.uuid4().hex} + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16))) + assert [r.status_code for r in results] == [201]*16 + assert all(r.json() == results[0].json() for r in results) + assert client.get(path,headers=headers).json()['items'] == [results[0].json()] + + +def test_scope_and_replay_rechecks_authorization(client): + b, bb = user(client) + bh, _ = login(client, bb) + a, ah, path = setup_chat(client, [b['id']]) + key = 'same-in-independent-scopes' + first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json() + second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json() + other_path = '/api/v1/chats/'+chat['id']+'/messages' + third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}) + assert third.status_code == 201 and third.json()['id'] != first['id'] + outsider, ob = user(client) + oh, _ = login(client, ob) + assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404 + assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204 + assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401 + + +def test_expired_key_can_be_reused(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json() + with client.app.state.db.pool.connection() as conn: + ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds'] + assert 86390 <= ttl <= 86400 + conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key)) + second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + assert len(client.get(path,headers=h).json()['items']) == 2 + + +def test_rollback_side_effect_and_key_on_database_error(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + text = 'fault_' + uuid.uuid4().hex + function = 'fault_' + uuid.uuid4().hex + trigger = 'trigger_' + uuid.uuid4().hex + db = client.app.state.db + # AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook. + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text))) + conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert failed.status_code == 500 + assert failed.headers['X-Instance-Id'] + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0 + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert retried.status_code == 201 + assert client.get(path,headers=h).json()['items'] == [retried.json()] + + +def test_ready_requires_compatible_schema(client): + temporary = 'temporarily_unavailable_' + uuid.uuid4().hex + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary))) + try: + r = client.get('/health/ready') + assert r.status_code == 503 + assert r.json()['error']['code'] == 'unavailable' + assert client.get('/health/live').status_code == 200 + finally: + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary))) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 00000000..f5dcd958 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,63 @@ +import io +import pytest +from messenger.jobs import work, replay +from test_attachments import actors,upload,image_bytes,finish +from test_api import client + + +def job_for(client,aid): + with client.app.state.db.pool.connection() as conn: + return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id'] + + +def test_duplicate_ready_is_noop_and_not_replayable(client): + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + path='/api/v1/attachments/'+att['id'] + before=client.get(path,headers=ah).json() + job=job_for(client,att['id']) + for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1) + assert client.get(path,headers=ah).json()==before + with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready') + with client.app.state.db.pool.connection() as conn: + row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone() + assert row['attempt']==1 + assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None + + +def test_old_delivery_does_not_consume_attempt(client): + # Direct invocation avoids consumer timing; it exercises the same real DB/S3 code. + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + job=job_for(client,att['id']) + assert work(client.app.state.db,client.app.state.storage,job,999) + with client.app.state.db.pool.connection() as conn: + row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone() + assert row['attempt'] in (0,1) # independent live consumer may already finish this job + finish(client,att['id']) + + +def test_replay_sql_failure_releases_session_lock(client): + import psycopg + from messenger.db import setting + from psycopg.rows import dict_row + ah,_,_,chat=actors(client) + att=upload(client,ah,chat).json() + finish(client,att['id']) + job=job_for(client,att['id']) + db=client.app.state.db + with db.pool.connection() as conn: + conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id'])) + conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$") + conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()') + try: + with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock') + with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other: + acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok'] + if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),)) + assert acquired, 'Aborted replay leaked a session advisory lock into the pool' + finally: + with db.pool.connection() as conn: + conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays') + conn.execute('DROP FUNCTION lab5_fail_audit()') diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 00000000..eb492df8 --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,72 @@ +"""Real image transformations, no mock storage.""" +import hashlib +import io +import json +import subprocess +import sys +from PIL import Image + + +def test_worker_variants_and_original(tmp_path): + source=tmp_path/'input' + original=Image.new('RGBA',(320,200),(30,90,150,70)) + original.save(source,format='PNG') + before=hashlib.sha256(source.read_bytes()).hexdigest() + result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True) + assert result.returncode==0, result.stderr + manifest=json.loads(result.stdout) + assert hashlib.sha256(source.read_bytes()).hexdigest()==before + for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]: + item=manifest[kind] + with Image.open(tmp_path/kind) as image: + assert image.format==fmt and image.size==size + assert not image.getexif() + raw=(tmp_path/kind).read_bytes() + assert item['sha256']==hashlib.sha256(raw).hexdigest() + assert item['size_bytes']==len(raw) + + +def test_no_upscale_thumbnail_and_exif(tmp_path): + source=tmp_path/'input' + image=Image.new('RGB',(80,40),'red') + exif=Image.Exif();exif[274]=6 + image.save(source,format='JPEG',exif=exif) + result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True) + assert result.returncode==0, result.stderr + with Image.open(tmp_path/'thumbnail') as output: + assert output.size==(40,80) + assert not output.getexif() + + +def test_deadline_covers_slow_s3(monkeypatch): + import contextlib + import http.server + import os + import threading + import time + import uuid + import pytest + from messenger.jobs import process_objects + class Slow(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length','100') + self.end_headers() + try: + for _ in range(100): + self.wfile.write(b'x');self.wfile.flush();time.sleep(.05) + except (BrokenPipeError,ConnectionResetError): pass + def log_message(self,*args): pass + server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow) + thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start() + monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False) + monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port)) + monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2') + row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow', + 'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'} + started=time.monotonic() + try: + with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row) + assert time.monotonic()-started<4 + finally: + server.shutdown();server.server_close()