diff --git a/.env.example b/.env.example index 119384b5..28a38288 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,14 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +API_CPU_LIMIT=1.0 +API_MEMORY_LIMIT=512M +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +MAX_CONCURRENT_UPLOADS=10 +WORKER_CONCURRENCY=1 +WORKER_REPLICAS=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +RECONCILE_INTERVAL_SECONDS=900 +RECONCILE_GRACE_SECONDS=3600 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..be991447 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +CMD ["python3", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..c9c1d9c8 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,11 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults test-pipeline-replay test-outbox-recovery test-worker-recovery \ + test-reconcile-integration test-upload-concurrency load test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check/validate: offline contract checks; test: public smoke; student targets: focused attachment/pipeline checks" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +19,36 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachments: + $(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency-race: + $(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + $(PYTHON) student-tests/test_reconcile.py + +test-worker: + $(PYTHON) student-tests/test_worker.py + +test-pipeline-faults: + $(PYTHON) student-tests/test_pipeline_faults.py + +test-pipeline-replay: + $(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-outbox-recovery: + $(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-worker-recovery: + $(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile-integration: + $(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-upload-concurrency: + $(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index e98f75e9..6b4c0f84 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,86 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Группа и фамилия: BVT2304, Akkuratov. +- Целевая оценка: 5. +- Выбранный трек: 2 — доказанная идемпотентность pipeline. +- Commit/PR: не создан. +- Использованные библиотеки: psycopg, redis, boto3, Pillow, pika; версии закреплены в `requirements.txt`. +- В работе использовался ИИ-ассистент для переноса, адаптации и проверки кода. +- Проверенный стенд: Python 3.12.7; PostgreSQL 16.4; RabbitMQ 3.13.7; SeaweedFS 4.51; HAProxy 3.0.7; Docker Desktop 28.0.4; 2 API-реплики. Хост: Apple M2 Pro (10 ядер, 16 GiB RAM), macOS 27.0.1. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Python 3.9+ и Docker Compose v2. Полные команды есть в [code.md](./code.md). Основной запуск: + +```sh +cp .env.example .env +scripts/bootstrap-secrets.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done +make test BASE_URL=http://localhost:8080 +``` + +Секреты создаются локально и не публикуются. Миграции выполняются сервисом `migrate`; остановка без удаления данных — `docker compose down`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +PostgreSQL — источник истины для предметных данных, состояния вложения и transactional outbox. Подтверждённая API-загрузка сохраняет исходник в S3 staging и метаданные с outbox-записью до ответа `202 queued`. Publisher передаёт durable RabbitMQ job с ID и служебными метаданными, без bytes и credentials. Worker декодирует изображение в отдельном процессе с ограничением времени, записывает оригинал и `thumbnail`/`crop`/`optimized`, затем фиксирует `ready`. Повторы ограничены, используют backoff с jitter; lease и publisher восстанавливают просроченную работу. -## Доказательства критериев +Обработка ограничена `JOB_TIMEOUT_SECONDS`; дочерний процесс принудительно завершается при превышении. S3-клиент задаёт connect/read timeout и число retries, lease включает отдельный сетевой бюджет. Worker имеет memory limit 1 GiB и concurrency по умолчанию 1; API ограничивает файл 10 MiB и изображение 40 млн пикселей. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Версии pipeline задают отдельные параметры и S3-prefix для вариантов: v1 использует JPEG quality 85/WebP 80, v2 — JPEG 80/WebP 75. Геометрия остаётся одинаковой. Операторский replay создаёт новый `job_id`, сбрасывает попытку, публикует новую outbox-запись и добавляет audit event. SQL completion/failure ограничены текущими job ID, pipeline version и attempt; поздняя задача не может изменить текущее состояние. Reconciler периодически сверяет PostgreSQL и S3. Filer доступен только из внутренней Compose-сети. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Трек 2: гипотеза и методика -## Эксперименты и отказы +**Гипотеза:** повторная доставка может повторить физические записи, но уникальные versioned object keys и fencing по `(attachment_id, job_id, pipeline_version, attempt)` не допускают потери принятой работы или публикации устаревшего результата. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Методика: `make test-pipeline-faults` запускает fault injection на штатном consumer/publisher-коде и контролируемых брокере, объектном хранилище и state-store. Используется публичный синтетический fixture `sample.png`. Падения вводятся до первой побочной записи, после записи оригинала и после фиксации результата до ack; дополнительно проверяются повторы, reorder, смена версии и неопределённый publisher confirm. Результат каждого запуска записывается в [evidence/pipeline-faults.json](./evidence/pipeline-faults.json). -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Сценарий | Наблюдение | +| --- | --- | +| Дубль после commit до ack | 2 доставки, 4 записи объектов для одного набора; повторная доставка — 0 дополнительных записей | +| Crash сразу после claim | lease recovery создал attempt 2; повтор завершился `ready` | +| Crash сразу после записи оригинала | повторная запись детерминированна; attempt 2 завершился `ready` | +| Reorder attempt 2 затем attempt 1 | устаревшая попытка: 0 дополнительных записей | +| v1 superseded версией v2 во время обработки | metadata ссылается только на `v2` variants | +| Publisher crash до publish | pending outbox пережил перезапуск, задание опубликовано после восстановления | +| Confirm потерян после broker accept | два сообщения имеют идентичное тело и стабильный message ID; worker no-op для дубля | +| Ограничение времени transform | зависший дочерний процесс завершён за 1.008 секунды при бюджете 1 секунда | -## Ограничения и следующие шаги +Compose integration PASS: `make test-outbox-recovery` подтвердил переход queued→ready после принятого upload при остановленном publisher; `make test-pipeline-replay` выполнил v1→v2, повторно обработал failed-вложение на той же версии и проверил два audit-события. Исходный checksum сохранился. `make test-worker-recovery` подтвердил queued-вложение после перезапуска API/worker. Фактические результаты проверок стенда сохранены в [evidence/lab5-integration.json](./evidence/lab5-integration.json). -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Нагрузка + +Повторяемый замер: `.venv/bin/python tests/load.py --base-url http://localhost:18080 --duration 30 --concurrency 4 --output evidence/load.json`. После готовности API и подготовки тестовых клиентов выполнен 30-секундный closed-loop POST-тест на синтетических сообщениях: 18,768 успешных запросов, 0 ошибок, 625.458 успешных RPS; latency успешных запросов — p50 5.816 ms, p95 8.820 ms, p99 19.155 ms. Дополнительно `docker stats` измерял CPU и память контейнеров Compose: сумма CPU peak/mean — 322.07%/232.49%, сумма памяти peak/mean — 890,263,039/814,641,644 байт, 11 выборок (примерно раз в 2.73 секунды). API: две реплики, лимит каждой 1 CPU/512 MiB; worker memory limit 1 GiB. Отдельная предварительная фаза прогрева не проводилась. Это closed-loop результат на указанном хосте; его нельзя напрямую трактовать как open-loop capacity или гарантированный production throughput. Полные данные: [evidence/load.json](./evidence/load.json). + +## Инварианты и доставка + +- `202` означает, что исходник и намерение обработки устойчиво сохранены; publisher можно запускать позже. +- Job повторяется at-least-once. После потери confirm одно сообщение может быть опубликовано дважды; тест намеренно наблюдает две доставки. +- Результаты представлены детерминированными S3 keys, различными для pipeline versions; metadata переключается только условным SQL update. +- Ack отправляется только после устойчивого результата или после записи долговременного retry/outbox намерения. Дубль уже завершённой задачи — no-op. +- Это не exactly-once delivery: между broker accept и фиксацией publisher confirm возможна повторная публикация, а между SQL commit результата и consumer ack возможна повторная доставка. Гарантируется fencing/idempotency эффекта, не единственная доставка или единственное выполнение. + +## Проверки + +| Проверка | Команда | Результат | +| --- | --- | --- | +| Контракт skeleton | `make check` | PASS | +| OpenAPI/JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS | +| Варианты v1/v2 | `make test-worker PYTHON=.venv/bin/python` | PASS | +| Reconciler unit tests | `make test-reconcile PYTHON=.venv/bin/python` | PASS | +| Track 2 injection | `make test-pipeline-faults PYTHON=.venv/bin/python` | PASS; 8 сценариев | +| Python syntax/whitespace | `compileall`, `git diff --check` | PASS | +| Compose parse | `docker compose --env-file .env.example config --quiet` | PASS | +| Public HTTP smoke | `make test BASE_URL=http://localhost:18080` | PASS | +| Ошибочные загрузки/ACL | `make test-attachments BASE_URL=http://localhost:18080` | PASS | +| Concurrent idempotency | `make test-idempotency-race BASE_URL=http://localhost:18080` | PASS; 12 запросов, 2 API, одно сообщение | +| Outbox после publisher stop | `make test-outbox-recovery BASE_URL=http://localhost:18080` | PASS; queued→ready без ручной публикации | +| Очередь после API/worker restart | `make test-worker-recovery BASE_URL=http://localhost:18080` | PASS; ссылка сообщения и оригинал сохранены | +| Реальный replay v1→v2 | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; разные thumbnail SHA-256, оригинал совпал | +| Повтор failed-вложения и audit | `make test-pipeline-replay BASE_URL=http://localhost:18080` | PASS; failed→queued→ready, два audit-события | +| Reconciliation/S3 ACL | `make test-reconcile-integration BASE_URL=http://localhost:18080` | PASS; orphan cleanup, exact restore, анонимный GET запрещён | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080` | PASS; данные и сессия пережили restart | +| API failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080` | PASS; 131 запись, восстановление за 0.681 s | +| Параллельная загрузка | `make test-upload-concurrency PYTHON=.venv/bin/python BASE_URL=http://localhost:18080` | PASS; 10 × 5,883,411 байт | diff --git a/code.md b/code.md new file mode 100644 index 00000000..18dc92ae --- /dev/null +++ b/code.md @@ -0,0 +1,103 @@ +# Запуск и проверка лабораторной 5 + +## Подготовка стенда + +Нужны Docker Compose v2 и доступный Docker daemon. Все команды выполняются из корня репозитория. + +```sh +cp .env.example .env +scripts/bootstrap-secrets.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +``` + +Секреты создаются локально в `secrets/` с правами владельца и не выводятся в терминал. Не добавляйте `.env` или `secrets/` в Git. Compose поднимает PostgreSQL, RabbitMQ, SeaweedFS, миграции, API, publisher, worker, периодический reconciler и HAProxy. + +Дождитесь готовности API и проверьте состояние сервисов: + +```sh +until curl --fail --silent http://localhost:8080/health/ready >/dev/null; do sleep 2; done +docker compose ps +curl --fail --silent http://localhost:8080/health/ready +``` + +Если порт `8080` занят, поменяйте `APP_PORT` в локальном `.env` (например, на `18080`) и используйте этот порт во всех командах ниже. + +## Проверки кода + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check PYTHON=.venv/bin/python +make validate PYTHON=.venv/bin/python +make test-worker PYTHON=.venv/bin/python +make test-reconcile PYTHON=.venv/bin/python +make test-pipeline-faults PYTHON=.venv/bin/python +``` + +`test-pipeline-faults` исполняет fault injection для дубля, reorder, падения после claim и после записи оригинала, потери ack, гонки старой/новой версии, остановки publisher до publish, потери publisher confirm и hard timeout обработки. Санитизированный JSON создаётся в `evidence/pipeline-faults.json`. + +## API и интеграционные проверки + +Стенд должен быть запущен. Тесты создают синтетические данные и не удаляют их; используйте учебный стенд. + +```sh +make test BASE_URL=http://localhost:8080 +make test-attachments BASE_URL=http://localhost:8080 +make test-idempotency-race BASE_URL=http://localhost:8080 +make test-reconcile-integration BASE_URL=http://localhost:8080 +make test-upload-concurrency BASE_URL=http://localhost:8080 +make load BASE_URL=http://localhost:8080 +# Exact 30-second/4-client benchmark, saving sanitized metrics: +.venv/bin/python tests/load.py --base-url http://localhost:8080 \ + --duration 30 --concurrency 4 --output evidence/load.json +``` + +Для проверок уровня 4/5: + +```sh +# Подтверждённая API загрузка при остановленном publisher должна дойти до ready после его старта. +make test-outbox-recovery BASE_URL=http://localhost:8080 + +# Upload/message при остановленном worker; restart API и worker, затем ready и download. +make test-worker-recovery BASE_URL=http://localhost:8080 + +# Готовое вложение переобрабатывается pipeline v2 с новой версией job. +make test-pipeline-replay BASE_URL=http://localhost:8080 +``` + +Pipeline v1 использует JPEG quality 85 и WebP quality 80; v2 — JPEG quality 80 и WebP quality 75. Геометрия вариантов остаётся фиксированной контрактом. При replay `job_id` меняется, попытки начинаются заново, версии вариантов получают разные S3-prefix (`variants//vN/`), а событие записывается в PostgreSQL. В рабочем контейнере явный replay выполняется так: + +```sh +docker compose exec --index 1 -T api python3 -m src.replay_attachment \ + --attachment-id "$ATTACHMENT_ID" --pipeline-version 2 +``` + +Перед запуском задайте `ATTACHMENT_ID` равным UUID существующего вложения из ответа API. +Для повторной попытки failed-вложения укажите его текущую версию; переход на уже опубликованную версию разрешён только для failed, повышение версии supersede-ит старую активную задачу. + +## Отказ и сохранность + +```sh +chmod +x scripts/restart.sh scripts/stop-one.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080 +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080 +docker compose up -d --scale api=2 api +``` + +Сценарий failover оставляет выбранный API-контейнер остановленным; последняя команда возвращает две реплики. Restart hook перезапускает PostgreSQL, SeaweedFS, RabbitMQ, API, publisher и worker без удаления volumes. + +## Диагностика и остановка + +```sh +docker compose logs -f api publisher worker rabbitmq reconciler +docker compose down +``` + +`docker compose down` сохраняет named volumes. Для намеренного удаления тестовой БД, очереди и S3-файлов: + +```sh +docker compose down --volumes +``` + +Эту команду выполняйте только если данные стенда больше не нужны. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..25b777db --- /dev/null +++ b/compose.yaml @@ -0,0 +1,352 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + secrets: + - source: rabbitmq_config + target: /etc/rabbitmq/conf.d/10-defaults.conf + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + restart: unless-stopped + + s3-core: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - server + - -dir=/data + - -filer=true + - -s3=false + volumes: + - seaweed_data:/data + healthcheck: + test: + - CMD-SHELL + - "curl --fail --silent http://127.0.0.1:8888/ >/dev/null" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + networks: + - seaweed_internal + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - s3 + - -filer=s3-core:8888 + - -config=/run/secrets/s3_config + - -port=8333 + - -port.https=0 + - -port.grpc=0 + - -port.iceberg=0 + - -port.lance=0 + - -concurrentFileUploadLimit=10 + - -concurrentUploadLimitMB=100 + - -readerCacheSizeMB=64 + - -autoCreateBucket=false + - -ip.bind=0.0.0.0 + secrets: + - s3_config + depends_on: + s3-core: + condition: service_healthy + networks: + - default + - seaweed_internal + healthcheck: + test: + - CMD-SHELL + - "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3-bootstrap: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/bootstrap_s3.py"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10} + UPLOAD_TIMEOUT_SECONDS: "60" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX_SIZE: "8" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + DB_LOCK_TIMEOUT_MS: "2000" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python3 + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" + interval: 3s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEMORY_LIMIT:-512M}" + stop_grace_period: 20s + restart: unless-stopped + + publisher: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.publisher"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + MAX_IMAGE_PIXELS: "40000000" + DB_POOL_MAX_SIZE: "2" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + reconciler: + build: + context: . + dockerfile: Dockerfile + command: ["sh", "scripts/reconcile-loop.sh"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.7-alpine + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_config: + file: ./secrets/s3.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + rabbitmq_password: + file: ./secrets/rabbitmq_password + rabbitmq_config: + file: ./secrets/rabbitmq.conf + broker_url: + file: ./secrets/broker_url + +volumes: + postgres_data: + seaweed_data: + rabbitmq_data: + +networks: + seaweed_internal: + internal: true diff --git a/evidence/lab5-integration.json b/evidence/lab5-integration.json new file mode 100644 index 00000000..24d27e9b --- /dev/null +++ b/evidence/lab5-integration.json @@ -0,0 +1,120 @@ +{ + "date": "2026-11-04", + "compose": { + "api_replicas": 2, + "host_port": 18080, + "services_healthy": true, + "readiness": "200" + }, + "fixture": { + "file": "tests/fixtures/sample.png", + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "size_bytes": 111817 + }, + "checks": [ + { + "name": "public_smoke", + "result": "PASS", + "command": "make test BASE_URL=http://localhost:18080" + }, + { + "name": "attachment_negative_cases", + "result": "PASS", + "command": "make test-attachments BASE_URL=http://localhost:18080", + "cases": [ + "damaged PNG", + "spoofed MIME", + "oversized upload", + "pixel limit", + "unsafe filename", + "unauthorized chat", + "cross-chat attachment" + ] + }, + { + "name": "idempotency_race", + "result": "PASS", + "replicas": 2, + "concurrent_same_key_requests": 12, + "messages_created": 1 + }, + { + "name": "outbox_recovery", + "result": "PASS", + "status_before_publisher_restart": "queued", + "status_after_restart": "ready", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + }, + { + "name": "worker_restart_recovery", + "result": "PASS", + "sequence": [ + "stop worker", + "upload returns 202 queued", + "link attachment to message", + "restart API", + "start worker", + "attachment reaches ready" + ], + "message_attachment_link_survived": true + }, + { + "name": "pipeline_version_replay", + "result": "PASS", + "version_transition": "1 -> 2", + "job_id_changed": true, + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "v1_thumbnail_sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "v2_thumbnail_sha256": "7904b78c14d936a9e5b4d024bd7bf9ffe4de546c484b356177056b8e949e763f" + }, + { + "name": "failed_pipeline_replay", + "result": "PASS", + "previous_job_id": "2628b822-ae4c-4784-b3bf-03f695176984", + "failed_replay_job_id": "797cc24f-0c3a-49fa-9f25-febd21506c6c", + "pipeline_version": 2, + "audited_replays": 2, + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + }, + { + "name": "reconciliation", + "result": "PASS", + "dataset_objects_scanned": 36, + "metadata_records_scanned": 9, + "scan_duration_ms": 20.873, + "orphan_detected_and_removed": 1, + "missing_object_detected_and_restored": 1, + "fresh_orphan_preserved_until_grace": true, + "anonymous_s3_access": "denied" + }, + { + "name": "persistence_restart", + "result": "PASS", + "confirmed_data_preserved": true, + "existing_session_preserved": true + }, + { + "name": "api_failover", + "result": "PASS", + "instances_before": 2, + "successful_writes": 131, + "failed_attempts": 1, + "first_recovery_seconds": 0.681, + "confirmed_data_preserved": true + }, + { + "name": "concurrent_upload_memory", + "result": "PASS", + "api_replicas": 2, + "uploads": 10, + "concurrency": 10, + "bytes_per_upload": 5883411, + "duration_seconds": 2.525, + "baseline_max_container_bytes": 96101990, + "peak_max_container_bytes": 96101990, + "peak_delta_bytes": 0, + "sampling_method": "continuous docker stats stream" + } + ], + "track_2_fault_injection": "pipeline-faults.json" +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..c39e0935 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,43 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.007, + "concurrency": 4, + "successful_requests": 18768, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 625.458, + "successful_latency_ms": { + "p50": 5.816, + "p95": 8.82, + "p99": 19.155 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "environment": { + "host": "Apple M2 Pro", + "host_cpu_cores": 10, + "host_ram_bytes": 17179869184, + "macos_version": "27.0.1", + "docker_desktop_version": "28.0.4", + "api_replicas": 2, + "api_cpu_limit_per_container": 1.0, + "api_memory_limit_per_container": "512M", + "worker_memory_limit": "1G" + }, + "warmup": "No separate warm-up interval; run started after readiness and test-client bootstrap.", + "dataset": "18,768 successful synthetic POST message writes; concurrency 4.", + "resource_sampling": { + "sample_interval_seconds_approx": 2.73, + "samples": 11, + "cpu_percent_sum_all_compose_containers": { + "peak": 322.07, + "mean": 232.49 + }, + "memory_bytes_sum_all_compose_containers": { + "peak": 890263039, + "mean": 814641644 + }, + "source": "docker stats --no-stream; aggregate across running Compose containers" + } +} diff --git a/evidence/pipeline-faults.json b/evidence/pipeline-faults.json new file mode 100644 index 00000000..64c64d0c --- /dev/null +++ b/evidence/pipeline-faults.json @@ -0,0 +1,58 @@ +{ + "fixture": "tests/fixtures/sample.png", + "invariants": { + "ambiguous_confirm_republishes_stable_job": true, + "confirmed_work_eventually_ready": true, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + "duplicate_effects_create_no_second_variant_set": true, + "stale_attempt_writes_no_effects": true, + "stale_pipeline_cannot_publish_current_metadata": true + }, + "result": "PASS", + "scenarios": [ + { + "deliveries": 2, + "duplicate_delivery_writes": 0, + "effective_object_writes": 4, + "scenario": "duplicate_delivery_after_commit_before_ack" + }, + { + "attempts": 2, + "scenario": "crash_after_claim", + "terminal_status": "ready" + }, + { + "attempts": 2, + "scenario": "crash_after_original_put", + "terminal_status": "ready" + }, + { + "current_attempt": 2, + "scenario": "out_of_order_attempt_delivery", + "stale_attempt_extra_writes": 0 + }, + { + "current_pipeline_version": 2, + "published_variant_versions": [ + "v2" + ], + "scenario": "stale_pipeline_after_newer_version" + }, + { + "outbox_recovered_after_restart": true, + "published_after_restart": 1, + "scenario": "publisher_crash_before_publish" + }, + { + "deliveries": 2, + "same_message_id": true, + "scenario": "publish_confirm_lost_after_broker_accept" + }, + { + "configured_timeout_seconds": 1, + "observed_seconds": 1.008, + "scenario": "transform_timeout_terminated" + } + ], + "track": "2" +} diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 00000000..9dedf30a --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + option http-keep-alive + timeout connect 2s + timeout client 15s + timeout server 15s + timeout http-request 5s + timeout check 1s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend messenger + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + option redispatch + default-server inter 1s fall 2 rise 2 + server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..f5431a5f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..aae5c1ae --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idempotency_key), + CONSTRAINT idempotency_method_post CHECK (method = 'POST'), + CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a4178a77 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,59 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank; + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + staging_key text NOT NULL UNIQUE, + job_id uuid NOT NULL UNIQUE, + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + lease_until timestamptz, + status varchar(16) NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'), + error_code varchar(64), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + attempt integer NOT NULL CHECK (attempt >= 1), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + UNIQUE (job_id, attempt) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, created_at) + WHERE published_at IS NULL; + +CREATE TABLE reconciliation_incidents ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')), + first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, incident_type) +); diff --git a/migrations/004_pipeline_replay.sql b/migrations/004_pipeline_replay.sql new file mode 100644 index 00000000..424dcc59 --- /dev/null +++ b/migrations/004_pipeline_replay.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachment_outbox + DROP CONSTRAINT attachment_outbox_job_id_fkey; + +CREATE TABLE attachment_pipeline_events ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + prior_job_id uuid NOT NULL, + job_id uuid NOT NULL, + prior_version integer NOT NULL CHECK (prior_version >= 1), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + action varchar(16) NOT NULL CHECK (action = 'replay'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_pipeline_events_attachment_created_idx + ON attachment_pipeline_events (attachment_id, created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..9cd5699d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 +boto3==1.35.99 +Pillow==11.0.0 +pika==1.3.2 diff --git a/scripts/bootstrap-secrets.py b/scripts/bootstrap-secrets.py new file mode 100644 index 00000000..d0147596 --- /dev/null +++ b/scripts/bootstrap-secrets.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import json +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" + + +def main(): + SECRET_DIR.mkdir(mode=0o700, exist_ok=True) + names = ( + "postgres_password", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + "s3_admin_access_key_id", + "s3_admin_secret_access_key", + "rabbitmq_password", + "rabbitmq.conf", + "broker_url", + "s3.json", + ) + existing = [name for name in names if (SECRET_DIR / name).exists()] + if existing: + raise SystemExit( + "Secret files already exist; refusing to rotate credentials without a migration: " + + ", ".join(existing) + ) + + db_password = secrets.token_hex(24) + broker_password = secrets.token_urlsafe(32) + values = { + "postgres_password": db_password, + "database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger", + "s3_access_key_id": "mngr" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(40), + "s3_admin_access_key_id": "admin" + secrets.token_hex(7), + "s3_admin_secret_access_key": secrets.token_urlsafe(40), + "rabbitmq_password": broker_password, + "broker_url": f"amqp://messenger:{broker_password}@rabbitmq:5672/%2f", + } + for name, value in values.items(): + path = SECRET_DIR / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + descriptor = os.open( + SECRET_DIR / "rabbitmq.conf", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write("default_user = messenger\n") + output.write(f"default_pass = {broker_password}\n") + s3_config = { + "identities": [ + { + "name": "bootstrap-admin", + "credentials": [ + { + "accessKey": values["s3_admin_access_key_id"], + "secretKey": values["s3_admin_secret_access_key"], + } + ], + "actions": ["Admin"], + }, + { + "name": "messenger-api", + "credentials": [ + { + "accessKey": values["s3_access_key_id"], + "secretKey": values["s3_secret_access_key"], + } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments", + ], + }, + ] + } + descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(s3_config, output, separators=(",", ":")) + output.write("\n") + print("Local secrets generated with owner-only permissions; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100755 index 00000000..5ea636bd --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 "$(dirname "$0")/bootstrap-secrets.py" diff --git a/scripts/reconcile-loop.sh b/scripts/reconcile-loop.sh new file mode 100644 index 00000000..cbd430f4 --- /dev/null +++ b/scripts/reconcile-loop.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +interval=${RECONCILE_INTERVAL_SECONDS:-900} +grace=${RECONCILE_GRACE_SECONDS:-3600} + +case "$interval" in + ''|*[!0-9]*) + echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +case "$grace" in + ''|*[!0-9]*) + echo "RECONCILE_GRACE_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +if [ "$interval" -lt 60 ]; then + echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2 + exit 2 +fi +if [ "$grace" -lt 60 ]; then + echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2 + exit 2 +fi + +while : +do + python3 src/reconcile.py --grace-seconds "$grace" --apply + sleep "$interval" +done diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..1d314244 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres s3-core rabbitmq +docker compose up -d --wait postgres s3-core rabbitmq +docker compose restart s3 +docker compose up -d --wait s3 +attempt=0 +until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 4 ]; then + echo "S3 object reads did not recover after restart" >&2 + exit 1 + fi + sleep 2 +done +docker compose restart api +docker compose restart publisher worker +docker compose up -d --wait api publisher worker diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..59e50fe8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +found= +for container_id in $(docker compose ps --quiet --status running api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + found=$container_id + break + fi +done + +if [ -z "$found" ]; then + printf '%s\n' 'No running API container matched the requested instance alias.' >&2 + exit 1 +fi +docker stop --time 5 "$found" >/dev/null diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..5bc8da31 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..d5229ba9 --- /dev/null +++ b/src/app.py @@ -0,0 +1,39 @@ +import logging +import os +import signal +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .store import Store + +LOGGER = logging.getLogger("messenger") + + +def run_server(): + store = Store() + store.open() + try: + handler = partial(MessengerHandler, store=store) + server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + run_server() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 00000000..3c3bc7a8 --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import logging +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger("messenger.s3.bootstrap") + + +def secret(name: str) -> str: + direct = os.getenv(name) + if direct: + return direct + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError(f"Set {name} or {name}_FILE") + + +def main(): + client = boto3.client( + "s3", + endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name="us-east-1", + aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"), + aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"), + config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}), + ) + bucket = os.getenv("S3_BUCKET", "messenger-attachments") + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + client.create_bucket(Bucket=bucket) + client.put_bucket_acl(Bucket=bucket, ACL="private") + LOGGER.info("private S3 bucket bootstrap completed") + + +if __name__ == "__main__": + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + main() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..2dadff58 --- /dev/null +++ b/src/config.py @@ -0,0 +1,61 @@ +import os +import re +import socket +import uuid +from pathlib import Path + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10")) +UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60")) +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") + + +def secret_value(name: str, default: str = "") -> str: + value = os.getenv(name) + if value: + return value + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + if default: + return default + raise RuntimeError(f"Set {name} or {name}_FILE") + + +S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY") +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if IDEMPOTENCY_TTL_SECONDS < 86400: + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") +if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024: + raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB") +if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000: + raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million") +if MAX_CONCURRENT_UPLOADS < 1: + raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive") +if UPLOAD_TIMEOUT_SECONDS < 1: + raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..de78501f --- /dev/null +++ b/src/errors.py @@ -0,0 +1,26 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass + + +class AttachmentNotReadyError(Exception): + pass + + +class ObjectMissingError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 00000000..4ce15c4f --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,560 @@ +import base64 +import hashlib +import json +import logging +import threading +import time +import tempfile +import uuid +import warnings +import zlib +from http.server import BaseHTTPRequestHandler +from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit +import unicodedata + +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +from psycopg_pool import PoolTimeout + +from .config import ( + IDEMPOTENCY_KEY_RE, + INSTANCE_ID, + MAX_CONCURRENT_UPLOADS, + MAX_IMAGE_PIXELS, + MAX_INFLIGHT_REQUESTS, + MAX_REQUEST_BODY_BYTES, + MAX_UPLOAD_BYTES, + UPLOAD_TIMEOUT_SECONDS, + USER_RE, +) +from .errors import ( + ApiError, + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .security import hash_password, isoformat, valid_uuid +from .store import Store + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() +UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS) +IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1) + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def setup(self): + super().setup() + self.connection.settimeout(15) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "A required storage service is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST": + chat_id = path.rsplit("/", 2)[1] + self._handle_upload_attachment(chat_id, query, request_id) + elif path.startswith("/api/v1/attachments/") and method == "GET": + attachment_path = path.removeprefix("/api/v1/attachments/") + if attachment_path.endswith("/content"): + attachment_id = attachment_path.removesuffix("/content") + self._handle_download_attachment(attachment_id, query, request_id) + else: + self._handle_get_attachment(attachment_path, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except IdempotencyConflictError: + self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id) + except AttachmentNotReadyError: + self._send_error(409, "conflict", "Attachment is not ready", request_id) + except ObjectMissingError: + LOGGER.error("attachment object missing for request %s", request_id) + self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id) + except (BotoCoreError, ClientError): + LOGGER.warning("object storage unavailable for request %s", request_id) + self._send_error(503, "unavailable", "Object storage is unavailable", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + idempotency_key = self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body: + raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids") + text = body["text"] + attachment_ids = body.get("attachment_ids", []) + if ( + not isinstance(text, str) + or len(text) > 4000 + or not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + or (not text.strip() and not attachment_ids) + ): + raise ApiError(400, "invalid_request", "Invalid message text") + attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids] + canonical_body = json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + message = self.store.create_idempotent_message( + chat_id, + actor["id"], + text, + attachment_ids, + idempotency_key, + canonical_body, + ) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]: + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + length = self.headers.get("Content-Length") + if length is None or self.headers.get("Transfer-Encoding"): + raise ApiError(400, "invalid_request", "A bounded Content-Length is required") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size <= 0: + raise ApiError(415, "unsupported_media_type", "Image is empty") + if size > MAX_UPLOAD_BYTES: + self.close_connection = True + remaining = min(size, MAX_UPLOAD_BYTES + 1) + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + break + self.connection.settimeout(min(15, remaining_time)) + try: + chunk = self.rfile.read(min(64 * 1024, remaining)) + except (TimeoutError, OSError): + LOGGER.info("stopped bounded drain of oversized upload") + break + if not chunk: + break + remaining -= len(chunk) + raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit") + content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + digest = hashlib.sha256() + bytes_read = 0 + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + try: + while bytes_read < size: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise ApiError(408, "invalid_request", "Image upload timed out") + self.connection.settimeout(min(15, remaining_time)) + chunk = self.rfile.read(min(64 * 1024, size - bytes_read)) + if not chunk: + raise ApiError(400, "invalid_request", "Incomplete image body") + content.write(chunk) + digest.update(chunk) + bytes_read += len(chunk) + except (TimeoutError, OSError): + content.close() + raise ApiError(400, "invalid_request", "Incomplete image body") from None + + try: + content.seek(0) + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + if actual_type is None or actual_type != declared_type: + raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type") + width, height = image.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") + image.verify() + content.seek(0) + with IMAGE_DECODE_GUARD: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + image.load() + content.seek(0) + return content, actual_type, size, digest.hexdigest() + except ApiError: + content.close() + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + content.close() + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error): + content.close() + raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None + + @staticmethod + def _valid_filename(filename: str) -> bool: + return ( + 1 <= len(filename) <= 255 + and not any(char in ("/", "\\", "\x00") for char in filename) + and not any(unicodedata.category(char) == "Cc" for char in filename) + ) + + def _handle_upload_attachment( + self, chat_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + if not self.store.can_access_chat(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Chat not found") + filenames = query.get("filename", []) + if len(filenames) != 1 or not self._valid_filename(filenames[0]): + raise ApiError(400, "invalid_request", "Invalid filename") + if not UPLOAD_GUARD.acquire(blocking=False): + raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted") + try: + content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type()) + try: + attachment = self.store.create_attachment( + chat_id, + actor["id"], + filenames[0], + content_type, + content, + size, + digest, + ) + finally: + content.close() + finally: + UPLOAD_GUARD.release() + if attachment is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(202, attachment, request_id) + + def _handle_get_attachment(self, attachment_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"]) + if attachment is None: + raise ApiError(404, "not_found", "Attachment not found") + self._send_json(200, attachment, request_id) + + def _handle_download_attachment( + self, attachment_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + variants = query.get("variant", ["original"]) + if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"): + raise ApiError(400, "invalid_request", "Invalid attachment variant") + variant = variants[0] + if variant != "original" and variant not in ("thumbnail", "crop", "optimized"): + raise ApiError(404, "not_found", "Attachment variant not found") + result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant) + if result is None: + raise ApiError(404, "not_found", "Attachment not found") + content_type, content = result + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(content))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Cache-Control", "private, no-store") + self.end_headers() + self.wfile.write(content) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..b494c642 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,50 @@ +import json +import os +from pathlib import Path + +import pika + +QUEUE_NAME = "attachments.process.v1" +DEAD_QUEUE_NAME = "attachments.process.dead" +LEASE_NETWORK_BUDGET_SECONDS = 150 + + +def broker_url() -> str: + value = os.getenv("BROKER_URL") + if value: + return value + file_path = os.getenv("BROKER_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set BROKER_URL or BROKER_URL_FILE") + + +def connect() -> pika.BlockingConnection: + parameters = pika.URLParameters(broker_url()) + parameters.connection_attempts = 3 + parameters.retry_delay = 1 + parameters.socket_timeout = 5 + parameters.blocked_connection_timeout = 10 + return pika.BlockingConnection(parameters) + + +def declare_queues(channel: pika.adapters.blocking_connection.BlockingChannel) -> None: + channel.queue_declare( + queue=DEAD_QUEUE_NAME, + durable=True, + arguments={"x-queue-type": "quorum", "x-quorum-initial-group-size": 1}, + ) + channel.queue_declare( + queue=QUEUE_NAME, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-quorum-initial-group-size": 1, + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": DEAD_QUEUE_NAME, + }, + ) + + +def encode_job(job: dict) -> bytes: + return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1ae27009 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 00000000..b49c67f4 --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,76 @@ +import logging +from datetime import datetime +from typing import BinaryIO, Iterator, Optional, Tuple, Union + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY +from .errors import ObjectMissingError + +LOGGER = logging.getLogger("messenger.s3") + + +class ObjectStore: + def __init__(self): + self.client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + + def put( + self, + key: str, + content: Union[bytes, BinaryIO], + content_type: str, + size_bytes: Optional[int] = None, + ) -> None: + if size_bytes is None: + size_bytes = len(content) + self.client.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=content, + ContentLength=size_bytes, + ContentType=content_type, + ) + + def get(self, key: str, max_bytes: int) -> bytes: + try: + response = self.client.get_object(Bucket=S3_BUCKET, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("NoSuchKey", "404", "NotFound"): + raise ObjectMissingError from exc + raise + body = response["Body"] + try: + value = body.read(max_bytes + 1) + finally: + body.close() + if len(value) > max_bytes: + raise RuntimeError("Stored object exceeds configured safety limit") + return value + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=S3_BUCKET, Key=key) + + def ready(self) -> bool: + self.client.head_bucket(Bucket=S3_BUCKET) + return True + + def objects(self) -> Iterator[Tuple[str, datetime]]: + pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET) + for page in pages: + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] diff --git a/src/pipeline_versions.py b/src/pipeline_versions.py new file mode 100644 index 00000000..5b87cfa4 --- /dev/null +++ b/src/pipeline_versions.py @@ -0,0 +1,4 @@ +PIPELINE_QUALITY = { + 1: {"jpeg": 85, "webp": 80}, + 2: {"jpeg": 80, "webp": 75}, +} diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 00000000..471cbff6 --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import logging +import os +import time +import uuid + +import pika +import psycopg + +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues, encode_job +from .security import isoformat +from .store import Store + +LOGGER = logging.getLogger("messenger.publisher") + + +def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool: + with store.pool.connection() as conn: + with conn.transaction(): + row = store.claim_outbox(conn) + if row is None: + return False + payload = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": isoformat(row["created_at"]), + } + channel.basic_publish( + exchange="", + routing_key=QUEUE_NAME, + body=encode_job(payload), + properties=pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(row["id"]), + type="attachment.process.v1", + ), + mandatory=True, + ) + conn.execute( + "UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + return True + + +def main() -> None: + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + store = Store() + store.open() + try: + while True: + connection = None + try: + connection = connect() + channel = connection.channel() + declare_queues(channel) + channel.confirm_delivery() + store.recover_expired_attachment_jobs( + int(os.getenv("JOB_MAX_ATTEMPTS", "5")), + int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS, + ) + if not publish_pending(store, channel): + time.sleep(1) + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("outbox publishing cycle failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..421e046f --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import logging +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +LOGGER = logging.getLogger("messenger.reconcile") + + +def reconcile( + store: Store, + grace_seconds: int, + apply_changes: bool, + restore_dir: Optional[Path] = None, +): + started = time.monotonic() + metadata = store.reconciliation_snapshot() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + found = set() + orphans = [] + for key, last_modified in store.objects.objects(): + found.add(key) + if key not in metadata and last_modified <= cutoff: + orphans.append(key) + + missing = [ + row for key, row in metadata.items() + if key not in found and row.get("reconcile_type", "original") == "original" + ] + restore_candidates = [] + if restore_dir is not None: + root = restore_dir.resolve(strict=True) + for row in missing: + source = restore_dir / str(row["id"]) + try: + resolved = source.resolve(strict=True) + if ( + resolved.parent != root + or not resolved.is_file() + or resolved.stat().st_size != row["size_bytes"] + ): + continue + content = resolved.read_bytes() + except OSError: + continue + if ( + len(content) == row["size_bytes"] + and hashlib.sha256(content).hexdigest() == row["sha256"].strip() + ): + restore_candidates.append((row, content)) + + restored = 0 + if apply_changes: + for key in orphans: + store.objects.delete(key) + restorable_ids = set() + for row, content in restore_candidates: + store.objects.put(row["object_key"], content, row["content_type"]) + store.clear_missing_object(row["id"]) + restorable_ids.add(row["id"]) + restored += 1 + for row in missing: + if row["id"] not in restorable_ids: + store.record_missing_object(row["id"]) + for key, row in metadata.items(): + if key in found and row.get("reconcile_type", "original") == "original": + store.clear_missing_object(row["id"]) + + unresolved_missing = len(missing) - restored + return { + "mode": "apply" if apply_changes else "dry-run", + "grace_seconds": grace_seconds, + "objects_seen": len(found), + "metadata_seen": len({str(row["id"]) for row in metadata.values()}), + "orphan_objects": len([key for key in found if key not in metadata]), + "stale_orphan_candidates": len(orphans), + "deleted_orphans": len(orphans) if apply_changes else 0, + "missing_objects": len(missing), + "unresolved_missing_objects": unresolved_missing, + "restorable_missing_objects": len(restore_candidates), + "restored_missing_objects": restored, + "missing_incidents_recorded": len(missing) - restored if apply_changes else 0, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + } + + +def main(): + parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects") + parser.add_argument( + "--restore-dir", + type=Path, + help="optional trusted backup directory with exact original bytes named by attachment UUID", + ) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error("--grace-seconds must be at least 60") + + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + store = Store() + store.open() + try: + result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir) + print(json.dumps(result, sort_keys=True)) + if result["unresolved_missing_objects"]: + LOGGER.error( + "%s database records reference missing S3 objects; metadata was retained for manual recovery", + result["unresolved_missing_objects"], + ) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/replay_attachment.py b/src/replay_attachment.py new file mode 100644 index 00000000..62ba8bfd --- /dev/null +++ b/src/replay_attachment.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import argparse +import json +import uuid + +from .store import Store + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Explicitly replay a failed attachment or start a newer pipeline version." + ) + parser.add_argument("--attachment-id", required=True, type=uuid.UUID) + parser.add_argument( + "--pipeline-version", + required=True, + type=int, + help="pipeline version implemented by the deployed worker", + ) + args = parser.parse_args() + if args.pipeline_version < 1: + parser.error("--pipeline-version must be positive") + + store = Store() + store.open() + try: + result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version) + print(json.dumps(result, sort_keys=True)) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6ec821f7 --- /dev/null +++ b/src/security.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import secrets +import uuid +from datetime import datetime, timezone + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + salt_text = base64.urlsafe_b64encode(salt).decode("ascii") + hash_text = base64.urlsafe_b64encode(derived).decode("ascii") + return f"scrypt$32768$8$1${salt_text}${hash_text}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError, UnicodeError): + return False diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..60fd3d56 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..cee5415f --- /dev/null +++ b/src/store.py @@ -0,0 +1,835 @@ +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import secrets +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, BinaryIO, Dict, List, Optional, Tuple + +import redis +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + IDEMPOTENCY_TTL_SECONDS, + SESSION_TTL_SECONDS, +) +from .errors import ( + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .object_store import ObjectStore +from .pipeline_versions import PIPELINE_QUALITY +from .security import isoformat, utc_now, valid_uuid, verify_password + +LOGGER = logging.getLogger("messenger.store") + + +def database_url() -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + file_path = os.getenv("DATABASE_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE") + + +class Store: + """PostgreSQL metadata, S3 objects, and optional Redis session cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + self.objects = ObjectStore() + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + self.objects.ready() + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + "attachment_ids": attachment_ids or [], + } + + @staticmethod + def attachment(row: Dict[str, Any]) -> Dict[str, Any]: + variants = row.get("variants") or [] + public_variants = [ + { + name: variant[name] + for name in ("name", "content_type", "size_bytes", "width", "height", "sha256") + } + for variant in variants + ] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"].strip(), + "status": row.get("status", "queued"), + "variants": public_variants, + "error_code": row.get("error_code"), + "created_at": isoformat(row["created_at"]), + "updated_at": isoformat(row["updated_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + if not row or not verify_password(password, row["password_hash"]): + return None + return self.user(row) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_idempotent_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: List[str], + idempotency_key: str, + canonical_body: str, + ) -> Optional[Dict[str, Any]]: + request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() + lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") + lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + if attachment_ids: + attachments = conn.execute( + "SELECT id, status FROM attachments " + "WHERE chat_id = %s AND id = ANY(%s) FOR SHARE", + (chat_id, attachment_ids), + ).fetchall() + if len(attachments) != len(attachment_ids): + return None + if any(item["status"] == "failed" for item in attachments): + raise AttachmentNotReadyError + + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = conn.execute( + "SELECT request_hash, response, expires_at " + "FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " + "FOR UPDATE", + (sender_id, chat_id, idempotency_key), + ).fetchone() + if existing and existing["expires_at"] > utc_now(): + if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): + raise IdempotencyConflictError + return existing["response"] + if existing: + conn.execute( + "DELETE FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", + (sender_id, chat_id, idempotency_key), + ) + + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, ordinal) " + "VALUES (%s, %s, %s)", + (row["id"], attachment_id, ordinal), + ) + message = self.message(row, attachment_ids) + conn.execute( + "INSERT INTO idempotency_records " + "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " + "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False, separators=(",", ":")), + utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), + ), + ) + return message + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + ids_by_message = {} + if rows: + links = conn.execute( + "SELECT message_id, attachment_id FROM message_attachments " + "WHERE message_id = ANY(%s) ORDER BY message_id, ordinal", + ([row["id"] for row in rows],), + ).fetchall() + for link in links: + ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"])) + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor + + def can_access_chat(self, chat_id: str, user_id: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() is not None + + def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + content: BinaryIO, + size_bytes: int, + digest: str, + ) -> Optional[Dict[str, Any]]: + if not self.can_access_chat(chat_id, owner_id): + return None + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"original/{attachment_id}" + content.seek(0) + self.objects.put(staging_key, content, content_type, size_bytes) + try: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO attachments " + "(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, staging_key, job_id) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " + "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "status, variants, error_code, created_at, updated_at", + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + digest, + object_key, + staging_key, + job_id, + ), + ).fetchone() + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, 1, 1)", + (uuid.uuid4(), job_id, attachment_id), + ) + return self.attachment(row) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "attachment staging compensation failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + + def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + return self.attachment(row) if row else None + + def download_attachment( + self, attachment_id: str, user_id: str, variant: str = "original" + ) -> Optional[Tuple[str, bytes]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + if not row: + return None + if row["status"] != "ready": + raise AttachmentNotReadyError + if variant == "original": + key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"] + else: + match = next( + (item for item in (row["variants"] or []) if item["name"] == variant), + None, + ) + if match is None: + return None + key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"] + return content_type, self.objects.get(key, size_bytes) + + def claim_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int + ) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, " + "object_key, content_type, size_bytes, sha256 " + "FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + not row + or str(row["job_id"]) != job_id + or row["pipeline_version"] != pipeline_version + or row["status"] in ("ready", "failed") + or row["attempt"] != attempt + ): + return None + if row["status"] == "processing" and row["lease_until"] is not None: + if row["lease_until"] > utc_now(): + return None + return conn.execute( + "UPDATE attachments SET status = 'processing', " + "lease_until = clock_timestamp() + (%s * interval '1 second'), " + "updated_at = clock_timestamp() " + "WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, " + "size_bytes, sha256, attempt, pipeline_version", + (lease_seconds, attachment_id), + ).fetchone() + + def complete_attachment_job( + self, + attachment_id: str, + job_id: str, + pipeline_version: int, + attempt: int, + variants: List[Dict[str, Any]], + ) -> bool: + with self.pool.connection() as conn: + with conn.transaction(): + updated = conn.execute( + "UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, " + "lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s " + "AND status = 'processing' AND lease_until > clock_timestamp()", + ( + json.dumps(variants, separators=(",", ":")), + attachment_id, + job_id, + pipeline_version, + attempt, + ), + ) + return updated.rowcount == 1 + + def fail_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, + max_attempts: int, error_code: str, retry_delay_seconds: Optional[float], + ) -> None: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT status, attempt FROM attachments " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE", + (attachment_id, job_id, pipeline_version), + ).fetchone() + if not row or row["status"] != "processing" or row["attempt"] != attempt: + return + if retry_delay_seconds is not None and attempt < max_attempts: + next_attempt = attempt + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, error_code, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt, available_at) " + "VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second')) " + "ON CONFLICT (job_id, attempt) DO NOTHING", + ( + uuid.uuid4(), job_id, attachment_id, pipeline_version, + next_attempt, retry_delay_seconds, + ), + ) + else: + final_code = "retry_exhausted" if attempt >= max_attempts else error_code + conn.execute( + "UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, " + "updated_at = clock_timestamp() WHERE id = %s", + (final_code, attachment_id), + ) + + def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]: + if pipeline_version not in PIPELINE_QUALITY: + raise ValueError("pipeline version is not implemented by this deployment") + with self.pool.connection() as conn: + current = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, " + "content_type, size_bytes, sha256 FROM attachments WHERE id = %s", + (attachment_id,), + ).fetchone() + if current is None: + raise ValueError("attachment not found") + if pipeline_version < current["pipeline_version"]: + raise ValueError("pipeline version cannot move backwards") + if pipeline_version == current["pipeline_version"] and current["status"] != "failed": + raise ValueError("same-version replay is allowed only for failed attachments") + if current["status"] not in ("queued", "processing", "ready", "failed"): + raise ValueError("attachment cannot be replayed in its current state") + + try: + content = self.objects.get(current["object_key"], current["size_bytes"]) + except ObjectMissingError as object_error: + try: + content = self.objects.get(current["staging_key"], current["size_bytes"]) + except ObjectMissingError: + raise object_error + if ( + len(content) != current["size_bytes"] + or hashlib.sha256(content).hexdigest() != current["sha256"].strip() + ): + raise ValueError("stored original does not match attachment metadata") + + prior_job_id = str(current["job_id"]) + next_job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}/{next_job_id}" + self.objects.put(staging_key, content, current["content_type"], len(content)) + try: + with self.pool.connection() as conn: + with conn.transaction(): + locked = conn.execute( + "SELECT job_id, pipeline_version, attempt, status FROM attachments " + "WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + locked is None + or str(locked["job_id"]) != prior_job_id + or locked["pipeline_version"] != current["pipeline_version"] + or locked["attempt"] != current["attempt"] + or locked["status"] != current["status"] + ): + raise RuntimeError("attachment changed while preparing replay") + conn.execute( + "UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, " + "status = 'queued', staging_key = %s, variants = '[]'::jsonb, " + "error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s", + (next_job_id, pipeline_version, staging_key, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, 1)", + (uuid.uuid4(), next_job_id, attachment_id, pipeline_version), + ) + conn.execute( + "INSERT INTO attachment_pipeline_events " + "(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) " + "VALUES (%s, %s, %s, %s, %s, 'replay')", + ( + attachment_id, + prior_job_id, + next_job_id, + current["pipeline_version"], + pipeline_version, + ), + ) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "replay staging cleanup failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + return { + "attachment_id": str(attachment_id), + "previous_job_id": prior_job_id, + "job_id": str(next_job_id), + "pipeline_version": pipeline_version, + "attempt": 1, + "status": "queued", + } + + def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + "SELECT id, job_id, pipeline_version, attempt FROM attachments " + "WHERE status = 'processing' AND lease_until < clock_timestamp() " + "ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100" + ).fetchall() + for row in rows: + if row["attempt"] >= max_attempts: + conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', " + "lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + continue + next_attempt = row["attempt"] + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, row["id"]), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING", + (uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt), + ) + return len(rows) + + def claim_outbox(self, conn): + return conn.execute( + "SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, " + "a.created_at FROM attachment_outbox o " + "JOIN attachments a ON a.id = o.attachment_id " + "WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() " + "ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1" + ).fetchone() + + def reconciliation_snapshot(self): + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, " + "size_bytes, sha256, content_type FROM attachments" + ).fetchall() + expected = {} + for row in rows: + if row["status"] == "ready": + expected[row["object_key"]] = {**row, "reconcile_type": "original"} + for variant in row["variants"] or []: + expected[variant["object_key"]] = { + **row, + "object_key": variant["object_key"], + "size_bytes": variant["size_bytes"], + "sha256": variant["sha256"], + "content_type": variant["content_type"], + "reconcile_type": "variant", + } + elif row["status"] in ("queued", "processing"): + expected[row["staging_key"]] = {**row, "reconcile_type": "staging"} + expected[row["object_key"]] = {**row, "reconcile_type": "inflight"} + for name, extension in ( + ("thumbnail", "jpg"), + ("crop", "jpg"), + ("optimized", "webp"), + ): + key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}" + expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"} + return expected + + def record_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO reconciliation_incidents (attachment_id, incident_type) " + "VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) " + "DO UPDATE SET last_seen_at = clock_timestamp()", + (attachment_id,), + ) + + def clear_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM reconciliation_incidents " + "WHERE attachment_id = %s AND incident_type = 'missing_object'", + (attachment_id,), + ) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..64298131 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +import hashlib +import io +import json +import logging +import multiprocessing +import os +import random +import time +import uuid +import warnings +import zlib +from typing import Callable, Optional + +import pika +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, ImageOps, UnidentifiedImageError + +from .config import MAX_IMAGE_PIXELS +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues +from .pipeline_versions import PIPELINE_QUALITY +from .store import Store + +LOGGER = logging.getLogger("messenger.worker") + +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1")) + + +class InvalidAttachment(Exception): + pass + + +class UnsupportedPipeline(InvalidAttachment): + pass + + +def _rgb_image(content: bytes) -> Image.Image: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(content)) as source: + if source.format not in ("PNG", "JPEG"): + raise InvalidAttachment("unsupported_image") + width, height = source.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise InvalidAttachment("image_too_large") + source.load() + image = ImageOps.exif_transpose(source) + try: + if "A" in image.getbands() or "transparency" in image.info: + with image.convert("RGBA") as rgba: + background = Image.new("RGB", rgba.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + finally: + if image is not source: + image.close() + except InvalidAttachment: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError, + UnidentifiedImageError, ValueError, zlib.error) as exc: + raise InvalidAttachment("invalid_image") from exc + + +def _encode(image: Image.Image, image_format: str, **options) -> bytes: + output = io.BytesIO() + image.save(output, format=image_format, **options) + return output.getvalue() + + +def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1): + quality = PIPELINE_QUALITY.get(pipeline_version) + if quality is None: + raise UnsupportedPipeline("unsupported_pipeline_version") + image = _rgb_image(content) + try: + variants = [] + thumbnail = image.copy() + try: + thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes)) + finally: + thumbnail.close() + + crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + try: + crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("crop", "image/jpeg", 128, 128, crop_bytes)) + finally: + crop.close() + + optimized = image.copy() + try: + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6) + variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes)) + finally: + optimized.close() + + result = [] + for name, content_type, width, height, data in variants: + extension = "webp" if content_type == "image/webp" else "jpg" + key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}" + result.append( + { + "name": name, + "content_type": content_type, + "size_bytes": len(data), + "width": width, + "height": height, + "sha256": hashlib.sha256(data).hexdigest(), + "object_key": key, + "content": data, + } + ) + return result + finally: + image.close() + + +class ProcessingTimeout(TimeoutError): + pass + + +def _variant_process(connection, attachment_id, content, pipeline_version, builder): + try: + connection.send(("ok", builder(attachment_id, content, pipeline_version))) + except InvalidAttachment as exc: + connection.send(("invalid", str(exc))) + except Exception as exc: + connection.send(("error", type(exc).__name__)) + finally: + connection.close() + + +def _build_variants_bounded( + attachment_id: str, + content: bytes, + pipeline_version: int, + timeout_seconds: int, + builder=build_variants, +): + if timeout_seconds < 1: + raise ValueError("timeout_seconds must be positive") + context = multiprocessing.get_context("spawn") + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_variant_process, + args=(sender, attachment_id, content, pipeline_version, builder), + ) + process.start() + sender.close() + try: + if not receiver.poll(timeout_seconds): + process.terminate() + process.join(timeout=2) + if process.is_alive(): + process.kill() + process.join() + raise ProcessingTimeout("image processing exceeded its time budget") + try: + result, value = receiver.recv() + except EOFError as exc: + raise RuntimeError("image processing process exited without a result") from exc + process.join(timeout=2) + if process.is_alive(): + process.terminate() + process.join() + if result == "invalid": + raise InvalidAttachment(value) + if result == "error": + raise RuntimeError(f"image processing failed in subprocess ({value})") + return value + finally: + receiver.close() + if process.is_alive(): + process.terminate() + process.join() + + +def _job(payload: bytes): + value = json.loads(payload) + required = { + "schema_version", "job_id", "type", "attachment_id", + "pipeline_version", "attempt", "created_at", + } + if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}): + raise ValueError("invalid job fields") + if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1": + raise ValueError("invalid job schema") + uuid.UUID(value["job_id"]) + uuid.UUID(value["attachment_id"]) + if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1: + raise ValueError("invalid pipeline version") + if not isinstance(value["attempt"], int) or value["attempt"] < 1: + raise ValueError("invalid job attempt") + return value + + +def _process( + store: Store, + job: dict, + fault_injector: Optional[Callable[[str], None]] = None, +) -> None: + def inject(point: str) -> None: + if fault_injector is not None: + fault_injector(point) + + attachment_id = job["attachment_id"] + row = store.claim_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS, + ) + if row is None: + LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"]) + return + + try: + inject("after_claim") + content = store.objects.get(row["staging_key"], row["size_bytes"]) + if hashlib.sha256(content).hexdigest() != row["sha256"].strip(): + raise InvalidAttachment("checksum_mismatch") + variants = _build_variants_bounded( + attachment_id, + content, + job["pipeline_version"], + JOB_TIMEOUT_SECONDS, + ) + inject("before_original_put") + store.objects.put(row["object_key"], content, row["content_type"], len(content)) + inject("after_original_put") + persisted = [] + for variant in variants: + data = variant.pop("content") + inject(f"before_variant_put:{variant['name']}") + store.objects.put(variant["object_key"], data, variant["content_type"], len(data)) + inject(f"after_variant_put:{variant['name']}") + persisted.append(variant) + inject("before_commit") + if not store.complete_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + persisted, + ): + LOGGER.warning("job lease was superseded before completion: %s", job["job_id"]) + return + inject("after_commit") + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id) + LOGGER.info("attachment processing completed: %s", attachment_id) + except InvalidAttachment as exc: + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, str(exc), None, + ) + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id) + LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc) + except Exception as exc: + delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25) + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, "processing_failed", delay, + ) + LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__) + + +def handle_delivery( + store: Store, + channel: pika.adapters.blocking_connection.BlockingChannel, + method, + body: bytes, + fault_injector: Optional[Callable[[str], None]] = None, +) -> None: + try: + job = _job(body) + except (ValueError, TypeError, json.JSONDecodeError): + LOGGER.exception("invalid broker job rejected") + channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + return + _process(store, job, fault_injector) + if fault_injector is not None: + fault_injector("before_ack") + channel.basic_ack(delivery_tag=method.delivery_tag) + + +def main() -> None: + if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1: + raise RuntimeError("Worker limits must be positive") + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + store = Store() + store.open() + connection = None + try: + while True: + try: + connection = connect() + channel = connection.channel() + declare_queues(channel) + channel.basic_qos(prefetch_count=WORKER_CONCURRENCY) + + def on_message(ch, method, properties, body): + handle_delivery(store, ch, method, body) + + channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False) + LOGGER.info("worker consuming durable attachment queue") + channel.start_consuming() + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("worker connection failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + connection = None + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..1663aaa3 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Negative attachment and cross-chat authorization checks against a running lab4 API.""" +import uuid +from pathlib import Path +import sys +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202): + return client.call( + "POST", + "/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""), + expected, + actor=actor, + raw=data, + headers={"Content-Type": content_type}, + schema="Attachment" if expected in (201, 202) else None, + ) + + +def main(): + args = arguments("Attachment upload, size, MIME, and chat authorization checks") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + other_chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + print("Checking accepted fixture upload.", flush=True) + attachment = upload(client, alice, chat["id"], image)["json"] + + print("Checking damaged and spoofed image rejection.", flush=True) + upload(client, alice, chat["id"], b"not an image", expected=415) + upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415) + print("Checking byte and pixel limits.", flush=True) + upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413) + import struct + import zlib + + def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30")) + bad_idat[-3] ^= 0xFF + damaged_idat = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", bytes(bad_idat)) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], damaged_idat, expected=415) + + too_many_pixels = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], too_many_pixels, expected=413) + print("Checking filename and cross-chat authorization.", flush=True) + upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400) + upload(client, eve, chat["id"], image, expected=404) + client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve) + client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + 404, + actor=alice, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": "", "attachment_ids": [attachment["id"]]}, + ) + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=bob, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": ""}, + ) + print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_idempotency_race.py b/student-tests/test_idempotency_race.py new file mode 100644 index 00000000..646869a7 --- /dev/null +++ b/student-tests/test_idempotency_race.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a concurrent idempotency-key race through the load balancer.""" +from concurrent.futures import ThreadPoolExecutor +import threading +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency integration check") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = "race-" + uuid.uuid4().hex + text = "concurrent-" + uuid.uuid4().hex + count = 12 + barrier = threading.Barrier(count) + + def send_once(_): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=10) + return worker.message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send_once, range(count))) + + instance_ids = {response["headers"]["X-Instance-Id"] for response in responses} + messages = [response["json"] for response in responses] + check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas") + check(all(message == messages[0] for message in messages), "same-key requests returned different messages") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"] + matches = [message for message in page["items"] if message["text"] == text] + check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages") + print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_outbox_recovery_integration.py b/student-tests/test_outbox_recovery_integration.py new file mode 100644 index 00000000..972228bc --- /dev/null +++ b/student-tests/test_outbox_recovery_integration.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify an upload committed while the publisher is stopped is processed after restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify committed attachment outbox intent survives publisher downtime") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + compose("stop", "publisher") + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted into queued state") + finally: + compose("start", "publisher") + + ready = client.wait_ready(alice, attachment) + stored_original = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + thumbnail = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + variant = next(item for item in ready["variants"] if item["name"] == "thumbnail") + check(stored_original == original, "recovered job changed original bytes") + check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch") + print( + json.dumps( + { + "result": "PASS", + "accepted_status_before_publisher_restart": attachment["status"], + "final_status": ready["status"], + "attachment_id": attachment["id"], + "original_sha256": hashlib.sha256(stored_original).hexdigest(), + "thumbnail_sha256": variant["sha256"], + "recovered_without_manual_republish": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 00000000..0d06e56a --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing.""" +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import uuid +from datetime import datetime, timezone + +os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret") + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import pika + +from src.publisher import publish_pending +from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery + + +ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001" +JOB_ID = "00000000-0000-4000-8000-000000000002" + + +class SimulatedCrash(BaseException): + pass + + +class FakeObjects: + def __init__(self, content): + self.data = {"staging/" + ATTACHMENT_ID: content} + self.puts = [] + + def get(self, key, max_bytes): + return self.data[key] + + def put(self, key, content, content_type, size_bytes=None): + value = content if isinstance(content, bytes) else content.read() + assert len(value) == size_bytes + self.data[key] = value + self.puts.append((key, hashlib.sha256(value).hexdigest())) + + def delete(self, key): + self.data.pop(key, None) + + +class FakeStore: + def __init__(self, content): + self.objects = FakeObjects(content) + self.row = { + "id": ATTACHMENT_ID, + "job_id": JOB_ID, + "pipeline_version": 1, + "attempt": 1, + "status": "queued", + "lease_until": 0.0, + "staging_key": "staging/" + ATTACHMENT_ID, + "object_key": "original/" + ATTACHMENT_ID, + "content_type": "image/png", + "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "variants": [], + } + + def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] in ("ready", "failed") + ): + return None + if row["status"] == "processing" and row["lease_until"] > time.monotonic(): + return None + row["status"] = "processing" + row["lease_until"] = time.monotonic() + lease_seconds + return dict(row) + + def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] != "processing" + or row["lease_until"] <= time.monotonic() + ): + return False + row["status"] = "ready" + row["lease_until"] = 0.0 + row["variants"] = variants + return True + + def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay): + row = self.row + if ( + row["id"] == attachment_id + and row["job_id"] == job_id + and row["pipeline_version"] == version + and row["attempt"] == attempt + and row["status"] == "processing" + ): + row["status"] = "failed" if delay is None else "queued" + row["lease_until"] = 0.0 + row["error_code"] = error_code + if delay is not None: + row["attempt"] += 1 + + def recover_expired(self): + if self.row["status"] == "processing": + self.row["attempt"] += 1 + self.row["status"] = "queued" + self.row["lease_until"] = 0.0 + + def replay(self, version): + row = self.row + previous_job = row["job_id"] + row["job_id"] = str(uuid.uuid4()) + row["pipeline_version"] = version + row["attempt"] = 1 + row["status"] = "queued" + row["lease_until"] = 0.0 + row["variants"] = [] + row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}" + self.objects.put( + row["staging_key"], + self.objects.data[row["object_key"]], + row["content_type"], + row["size_bytes"], + ) + return { + "schema_version": 1, + "job_id": row["job_id"], + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + "previous_job_id": previous_job, + } + + +class FakeChannel: + def __init__(self): + self.acked = [] + self.nacked = [] + + def basic_ack(self, delivery_tag): + self.acked.append(delivery_tag) + + def basic_nack(self, delivery_tag, requeue): + self.nacked.append((delivery_tag, requeue)) + + +class FakeMethod: + def __init__(self, delivery_tag): + self.delivery_tag = delivery_tag + + +def job_body(job_id=JOB_ID, version=1, attempt=1): + return json.dumps( + { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": attempt, + "created_at": "2026-01-01T00:00:00Z", + } + ).encode() + + +def crash_at(expected): + def inject(point): + if point == expected: + raise SimulatedCrash(point) + + return inject + + +def slow_builder(attachment_id, content, pipeline_version): + time.sleep(5) + + +def test_duplicate_and_ack_loss(content): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(1), job_body(), + fault_injector=crash_at("before_ack"), + ) + raise AssertionError("fault injection did not crash before ack") + except SimulatedCrash: + pass + assert store.row["status"] == "ready" + first_write_count = len(store.objects.puts) + assert channel.acked == [] + + handle_delivery(store, channel, FakeMethod(2), job_body()) + assert channel.acked == [2] + assert len(store.objects.puts) == first_write_count + assert store.row["status"] == "ready" + return { + "scenario": "duplicate_delivery_after_commit_before_ack", + "deliveries": 2, + "effective_object_writes": first_write_count, + "duplicate_delivery_writes": len(store.objects.puts) - first_write_count, + } + + +def test_crash_before_and_after_side_effect(content, crash_point): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(3), job_body(), + fault_injector=crash_at(crash_point), + ) + raise AssertionError(f"fault injection did not crash at {crash_point}") + except SimulatedCrash: + pass + assert channel.acked == [] + store.recover_expired() + retry = job_body(attempt=2) + handle_delivery(store, channel, FakeMethod(4), retry) + assert store.row["status"] == "ready" + assert store.row["attempt"] == 2 + assert channel.acked == [4] + assert store.objects.data["original/" + ATTACHMENT_ID] == content + return { + "scenario": "crash_" + crash_point, + "attempts": store.row["attempt"], + "terminal_status": store.row["status"], + } + + +def test_reordered_attempts(content): + store = FakeStore(content) + store.row["attempt"] = 2 + channel = FakeChannel() + handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2)) + writes_after_current = len(store.objects.puts) + handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1)) + assert store.row["status"] == "ready" + assert len(store.objects.puts) == writes_after_current + assert channel.acked == [5, 6] + return { + "scenario": "out_of_order_attempt_delivery", + "current_attempt": store.row["attempt"], + "stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current, + } + + +def test_stale_pipeline_cannot_publish_newer_result(content): + store = FakeStore(content) + channel = FakeChannel() + replayed = [] + + def supersede_after_first_variant(point): + if point == "after_variant_put:thumbnail" and not replayed: + replayed.append(store.replay(2)) + + handle_delivery( + store, channel, FakeMethod(7), job_body(), + fault_injector=supersede_after_first_variant, + ) + assert store.row["status"] == "queued" + assert store.row["pipeline_version"] == 2 + assert channel.acked == [7] + + current_job = replayed[0] + handle_delivery( + store, + channel, + FakeMethod(8), + job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]), + ) + assert store.row["status"] == "ready" + assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"} + assert channel.acked == [7, 8] + return { + "scenario": "stale_pipeline_after_newer_version", + "current_pipeline_version": store.row["pipeline_version"], + "published_variant_versions": sorted( + {variant["object_key"].split("/")[2] for variant in store.row["variants"]} + ), + } + + +class FakeTransaction: + def __init__(self, store): + self.store = store + + def __enter__(self): + self.published = False + self.store.current_transaction = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None and self.published: + self.store.outbox_published = True + self.store.current_transaction = None + return False + + +class FakeConnection: + def __init__(self, store): + self.store = store + + def transaction(self): + return FakeTransaction(self.store) + + def execute(self, query, params): + assert "UPDATE attachment_outbox" in query + assert params == ("outbox-row",) + self.store.current_transaction.published = True + + +class FakePool: + def __init__(self, store): + self.store = store + + def connection(self): + connection = FakeConnection(self.store) + + class ConnectionContext: + def __enter__(self_inner): + return connection + + def __exit__(self_inner, exc_type, exc_value, traceback): + return False + + return ConnectionContext() + + +class FakeOutboxStore: + def __init__(self): + self.pool = FakePool(self) + self.outbox_published = False + self.current_transaction = None + + def claim_outbox(self, connection): + if self.outbox_published: + return None + return { + "id": "outbox-row", + "job_id": JOB_ID, + "attachment_id": ATTACHMENT_ID, + "pipeline_version": 1, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + +class AmbiguousConfirmChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + self.messages.append(message) + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm") + + +class FailBeforePublishChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish") + self.messages.append(message) + + +def test_committed_outbox_survives_publisher_crash(): + store = FakeOutboxStore() + channel = FailBeforePublishChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher crash was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert channel.messages == [] + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 1 + return { + "scenario": "publisher_crash_before_publish", + "published_after_restart": len(channel.messages), + "outbox_recovered_after_restart": store.outbox_published, + } + + +def test_outbox_confirm_gap_republishes_same_job(): + store = FakeOutboxStore() + channel = AmbiguousConfirmChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher confirm fault was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 2 + first, second = channel.messages + assert first["body"] == second["body"] + assert first["properties"].message_id == second["properties"].message_id == "outbox-row" + return { + "scenario": "publish_confirm_lost_after_broker_accept", + "deliveries": len(channel.messages), + "same_message_id": True, + } + + +def test_image_processing_has_a_hard_timeout(): + started = time.monotonic() + try: + _build_variants_bounded( + ATTACHMENT_ID, + b"bounded timeout test", + 1, + timeout_seconds=1, + builder=slow_builder, + ) + raise AssertionError("slow transform exceeded its time budget without termination") + except ProcessingTimeout: + pass + elapsed = time.monotonic() - started + assert elapsed < 4 + return { + "scenario": "transform_timeout_terminated", + "configured_timeout_seconds": 1, + "observed_seconds": round(elapsed, 3), + } + + +def main(): + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + results = [ + test_duplicate_and_ack_loss(content), + test_crash_before_and_after_side_effect(content, "after_claim"), + test_crash_before_and_after_side_effect(content, "after_original_put"), + test_reordered_attempts(content), + test_stale_pipeline_cannot_publish_newer_result(content), + test_committed_outbox_survives_publisher_crash(), + test_outbox_confirm_gap_republishes_same_job(), + test_image_processing_has_a_hard_timeout(), + ] + report = { + "result": "PASS", + "track": "2", + "fixture": "tests/fixtures/sample.png", + "scenarios": results, + "invariants": { + "confirmed_work_eventually_ready": True, + "duplicate_effects_create_no_second_variant_set": True, + "stale_attempt_writes_no_effects": True, + "stale_pipeline_cannot_publish_current_metadata": True, + "ambiguous_confirm_republishes_stable_job": True, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + }, + } + evidence = ROOT / "evidence" / "pipeline-faults.json" + evidence.parent.mkdir(exist_ok=True) + evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, sort_keys=True)) + print(f"Evidence written to {evidence.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_pipeline_replay_integration.py b/student-tests/test_pipeline_replay_integration.py new file mode 100644 index 00000000..8687a954 --- /dev/null +++ b/student-tests/test_pipeline_replay_integration.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Verify explicit replay changes the deployed pipeline while preserving the original.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +MARK_FAILED_CODE = """ +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + updated = conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', " + "variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'", + (sys.argv[1],), + ) + if updated.rowcount != 1: + raise RuntimeError("test attachment was not ready for failed replay injection") +finally: + store.close() +""" + +AUDIT_CODE = """ +import json +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + rows = conn.execute( + "SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events " + "WHERE attachment_id = %s ORDER BY id", + (sys.argv[1],), + ).fetchall() + print(json.dumps(rows, sort_keys=True)) +finally: + store.close() +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def replay(container_id, attachment_id, version): + result = docker_exec( + container_id, + "python3", + "-m", + "src.replay_attachment", + "--attachment-id", + attachment_id, + "--pipeline-version", + str(version), + ) + return json.loads(result.stdout) + + +def main(): + args = arguments("Verify versioned attachment replay against the running Compose stack") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready_v1 = client.wait_ready(alice, attachment) + v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail") + v1_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch") + + containers = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.split() + check(bool(containers), "no running API container found") + replay_v2 = replay(containers[0], attachment["id"], 2) + check(replay_v2["status"] == "queued", "replay was not queued") + check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2") + check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id") + + ready_v2 = client.wait_ready(alice, ready_v1) + v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail") + v2_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding") + check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch") + check(original_bytes == original, "pipeline replay changed the original file") + + docker_exec( + containers[0], + "python3", + "-c", + MARK_FAILED_CODE, + attachment["id"], + ) + failed_replay = replay(containers[0], attachment["id"], 2) + check(failed_replay["status"] == "queued", "failed attachment replay was not queued") + check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version") + check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id") + ready_replayed = client.wait_ready(alice, ready_v2) + audit = json.loads( + docker_exec( + containers[0], + "python3", + "-c", + AUDIT_CODE, + attachment["id"], + ).stdout + ) + check( + [(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit] + == [(1, 2, "replay"), (2, 2, "replay")], + "pipeline replay audit trail is incomplete", + ) + check(ready_replayed["status"] == "ready", "failed attachment did not recover") + + print( + json.dumps( + { + "result": "PASS", + "previous_job_id": replay_v2["previous_job_id"], + "job_id": replay_v2["job_id"], + "pipeline_version": replay_v2["pipeline_version"], + "failed_replay_job_id": failed_replay["job_id"], + "audited_replays": len(audit), + "v1_thumbnail_sha256": v1_thumbnail["sha256"], + "v2_thumbnail_sha256": v2_thumbnail["sha256"], + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 00000000..ff123126 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Deterministic tests of the reconciliation decision rules without deleting real objects.""" +import os +import hashlib +import sys +import tempfile +from pathlib import Path + +os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret") +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from datetime import datetime, timedelta, timezone + +from src.reconcile import reconcile + + +class FakeObjects: + def __init__(self, items): + self.items = items + self.deleted = [] + + def objects(self): + return iter(self.items) + + def delete(self, key): + self.deleted.append(key) + self.items = [(name, date) for name, date in self.items if name != key] + + def put(self, key, content, content_type): + self.items.append((key, datetime.now(timezone.utc))) + + +class FakeStore: + def __init__(self): + now = datetime.now(timezone.utc) + self.metadata = { + "original/present": {"id": "present-id", "object_key": "original/present"}, + "original/missing": { + "id": "missing-id", + "object_key": "original/missing", + "size_bytes": 4, + "sha256": hashlib.sha256(b"test").hexdigest(), + "content_type": "image/png", + }, + } + self.objects = FakeObjects( + [ + ("original/present", now), + ("original/stale-orphan", now - timedelta(hours=2)), + ("original/new-orphan", now - timedelta(minutes=10)), + ] + ) + self.missing_incidents = set() + self.cleared_incidents = set() + + def reconciliation_snapshot(self): + return self.metadata + + def record_missing_object(self, attachment_id): + self.missing_incidents.add(attachment_id) + + def clear_missing_object(self, attachment_id): + self.missing_incidents.discard(attachment_id) + self.cleared_incidents.add(attachment_id) + + +def main(): + store = FakeStore() + dry_run = reconcile(store, grace_seconds=3600, apply_changes=False) + assert dry_run["stale_orphan_candidates"] == 1 + assert dry_run["missing_objects"] == 1 + assert dry_run["deleted_orphans"] == 0 + assert store.objects.deleted == [] and store.missing_incidents == set() + + applied = reconcile(store, grace_seconds=3600, apply_changes=True) + assert store.objects.deleted == ["original/stale-orphan"] + assert "original/new-orphan" not in store.objects.deleted + assert store.missing_incidents == {"missing-id"} + assert store.cleared_incidents == {"present-id"} + assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1 + + with tempfile.TemporaryDirectory() as backup: + restore_dir = Path(backup) + (restore_dir / "missing-id").write_bytes(b"test") + preview = reconcile(store, 3600, False, restore_dir) + assert preview["restorable_missing_objects"] == 1 + assert store.missing_incidents == {"missing-id"} + restored = reconcile(store, 3600, True, restore_dir) + assert restored["restored_missing_objects"] == 1 + assert store.missing_incidents == set() + + repeated = reconcile(store, grace_seconds=3600, apply_changes=True) + assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0 + assert store.objects.deleted == ["original/stale-orphan"] + assert store.missing_incidents == set() + print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_reconcile_integration.py b/student-tests/test_reconcile_integration.py new file mode 100644 index 00000000..78a719dd --- /dev/null +++ b/student-tests/test_reconcile_integration.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects.""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +STORE_CODE = """ +import sys +from pathlib import Path +from src.store import Store + +store = Store() +store.open() +try: + action = sys.argv[1] + if action == "key": + attachment_id = sys.argv[2] + matches = [ + key for key, row in store.reconciliation_snapshot().items() + if str(row["id"]) == attachment_id and row["reconcile_type"] == "original" + ] + if len(matches) != 1: + raise RuntimeError("attachment metadata lookup failed") + print(matches[0]) + elif action == "put": + key, file_path = sys.argv[2:4] + content = Path(file_path).read_bytes() + store.objects.put(key, content, "image/png", len(content)) + elif action == "delete": + store.objects.delete(sys.argv[2]) + elif action == "exists": + print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no") + else: + raise RuntimeError("unsupported test operation") +finally: + store.close() +""" + +PRIVATE_CHECK_CODE = """ +import socket +import sys +import urllib.error +import urllib.request + +try: + socket.getaddrinfo("s3-core", 8888) +except socket.gaierror: + print("PASS: SeaweedFS master/filer network is isolated") +else: + raise RuntimeError("API network can resolve the SeaweedFS core service") + +for port in (8181, 9101): + try: + socket.create_connection(("s3", port), timeout=1).close() + except OSError: + continue + raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port)) + +try: + urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3) +except urllib.error.HTTPError as error: + if error.code == 403: + print("PASS: unauthenticated bucket access denied") + else: + raise RuntimeError("unexpected anonymous S3 status: " + str(error.code)) +else: + raise RuntimeError("private bucket allowed an unauthenticated GET") +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def reconciliation(container_id, *args): + result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args) + return json.loads(result.stdout) + + +def object_exists(container_id, key): + return docker_exec( + container_id, "python3", "-c", STORE_CODE, "exists", key + ).stdout.strip() == "yes" + + +def main(): + args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + containers = result.stdout.split() + check(len(containers) >= 2, "at least two running API replicas are required") + container_id = containers[0] + key = docker_exec( + container_id, "python3", "-c", STORE_CODE, "key", attachment["id"] + ).stdout.strip() + orphan_key = "original/" + str(uuid.uuid4()) + + with tempfile.TemporaryDirectory() as backup: + source = Path(backup) / "original.png" + source.write_bytes(content) + container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png" + recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex + recovery_file = recovery_dir + "/" + attachment["id"] + orphan_created = False + orphan_removed = False + missing_deleted = False + missing_restored = False + try: + subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True) + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file) + orphan_created = True + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key) + missing_deleted = True + + preview = reconciliation(container_id) + check( + preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1, + "dry-run missed injected inconsistencies", + ) + check(preview["deleted_orphans"] == 0, "dry-run modified S3") + immediate_apply = reconciliation(container_id, "--apply") + check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period") + + docker_exec(container_id, "mkdir", "-p", recovery_dir) + subprocess.run( + ["docker", "cp", str(source), container_id + ":" + recovery_file], + check=True, + ) + restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir) + check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable") + restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored") + missing_restored = True + + repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object") + check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period") + print("Waiting for the 60-second orphan grace period.") + time.sleep(61) + + cleanup_preview = reconciliation(container_id) + check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace") + cleaned = reconciliation(container_id, "--apply") + orphan_removed = not object_exists(container_id, orphan_key) + check(orphan_removed, "injected stale orphan was not removed") + final_scan = reconciliation(container_id, "--apply") + check( + final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0, + "repeat scan is not clean", + ) + finally: + if missing_deleted and not missing_restored: + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file) + if orphan_created and not orphan_removed: + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key) + + private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip() + check("PASS:" in private, "unauthenticated S3 access check failed") + downloaded = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(downloaded == content, "reconciler changed the original of an attachment used by a message") + check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference") + print( + json.dumps( + { + "result": "PASS", + "dataset_objects_scanned": final_scan["objects_seen"], + "metadata_records_scanned": final_scan["metadata_seen"], + "scan_duration_ms": final_scan["duration_ms"], + "orphan_detected_and_removed": cleaned["deleted_orphans"], + "missing_detected_and_restored": restored["restored_missing_objects"], + "fresh_orphan_preserved_until_grace": True, + "message_attachment_download_checksum": "unchanged", + "private_bucket": "anonymous request denied", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker.py b/student-tests/test_worker.py new file mode 100644 index 00000000..8e922fac --- /dev/null +++ b/student-tests/test_worker.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Local transformation checks for the lab5 image worker.""" +import hashlib +import io +import os +from pathlib import Path +import sys + +os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret") + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.worker import build_variants + + +def main(): + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment_id = "00000000-0000-4000-8000-000000000001" + variants = build_variants(attachment_id, original, pipeline_version=1) + by_name = {item["name"]: item for item in variants} + assert set(by_name) == {"thumbnail", "crop", "optimized"} + assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160) + assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128) + assert by_name["optimized"]["content_type"] == "image/webp" + for item in variants: + content = item["content"] + assert len(content) == item["size_bytes"] + assert hashlib.sha256(content).hexdigest() == item["sha256"] + with Image.open(io.BytesIO(content)) as image: + assert image.size == (item["width"], item["height"]) + newer = build_variants(attachment_id, original, pipeline_version=2) + assert all(f"/v2/" in item["object_key"] for item in newer) + assert all(f"/v1/" in item["object_key"] for item in variants) + assert by_name["thumbnail"]["sha256"] != { + item["name"]: item for item in newer + }["thumbnail"]["sha256"] + print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_worker_recovery_integration.py b/student-tests/test_worker_recovery_integration.py new file mode 100644 index 00000000..e2cb5fe3 --- /dev/null +++ b/student-tests/test_worker_recovery_integration.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Verify queued uploads and message links survive an API/worker restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify queued attachment recovery after API and worker restart") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + compose("stop", "worker") + worker_stopped = True + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted while worker was stopped") + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message") + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + 409, + actor=bob, + ) + compose("restart", "api") + compose("start", "worker") + worker_stopped = False + client.call("GET", "/health/ready", schema="Health") + ready = client.wait_ready(bob, attachment) + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(original_bytes == original, "recovered worker changed original bytes") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages", + actor=bob, + schema="MessagePage", + )["json"] + recovered_message = next(item for item in page["items"] if item["id"] == message["id"]) + check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link") + print( + json.dumps( + { + "result": "PASS", + "initial_status": attachment["status"], + "final_status": ready["status"], + "message_attachment_link_survived": True, + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + finally: + if worker_stopped: + compose("start", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_concurrency.py b/student-tests/upload_concurrency.py new file mode 100644 index 00000000..e45746ca --- /dev/null +++ b/student-tests/upload_concurrency.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Run ten simultaneous near-6-MiB uploads and sample API container memory.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +import re +import subprocess +import threading +import time +import uuid +from io import BytesIO +from pathlib import Path +import sys + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} +USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/") + + +def usage_bytes(line): + match = USAGE_RE.search(line) + return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0 + + +def api_containers(): + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + ids = result.stdout.split() + check(len(ids) >= 2, "at least two running API containers are required") + return ids + + +def sample_once(ids): + output = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + values = [usage_bytes(line) for line in output] + check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage") + return max(values) + + +def main(): + args = arguments("Bounded-memory ten-client upload experiment") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + width = height = 1400 + content = BytesIO() + Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save( + content, format="PNG", compress_level=0 + ) + payload = content.getvalue() + check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit") + ids = api_containers() + baseline = sample_once(ids) + samples = [] + stats_process = subprocess.Popen( + ["docker", "stats", "--format", "{{.MemUsage}}", *ids], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def sampler(): + if stats_process.stdout is None: + return + for line in stats_process.stdout: + value = usage_bytes(line) + if value: + samples.append(value) + + sampling_thread = threading.Thread(target=sampler, daemon=True) + sampling_thread.start() + count = 10 + barrier = threading.Barrier(count) + + def upload(index): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=15) + return worker.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png", + 202, + actor=alice, + raw=payload, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + started = time.monotonic() + try: + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(upload, range(count))) + finally: + stopped_by_test = False + if stats_process.poll() is None: + stats_process.kill() + stopped_by_test = True + try: + stats_process.wait(timeout=5) + except subprocess.TimeoutExpired: + stats_process.kill() + stats_process.wait(timeout=5) + sampling_thread.join(timeout=5) + stats_error = stats_process.stderr.read() if stats_process.stderr is not None else "" + check( + stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9), + "docker stats sampling process failed (exit " + + str(stats_process.returncode) + + "): " + + stats_error.strip()[:300], + ) + check(bool(samples), "docker stats produced no memory samples") + peak = max([baseline] + samples) + check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed") + print( + json.dumps( + { + "uploads": count, + "concurrency": count, + "bytes_per_upload": len(payload), + "duration_seconds": round(time.monotonic() - started, 3), + "api_containers": len(ids), + "sample_count": len(samples) // len(ids), + "baseline_max_container_bytes": baseline, + "peak_max_container_bytes": peak, + "peak_delta_bytes": max(0, peak - baseline), + "sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence", + "memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main)