From 6196d4ee8e40a9ddc92c869e998ca4d9112aec6b Mon Sep 17 00:00:00 2001 From: kaktuz20000 <149298757+kaktuz20000@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:18:07 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 36 +- Dockerfile | 22 + Makefile | 39 +- README.md | 29 +- REPORT.md | 75 +- compose.example.yaml | 11 +- compose.yaml | 468 ++++++++++ evidence/incidents.json | 39 + evidence/pipeline-faults.json | 58 ++ evidence/track1-observability-cost.json | 208 +++++ haproxy.cfg | 35 + infra/otel/README.md | 10 +- infra/otel/alerts.yml | 42 + infra/otel/collector.yaml | 66 ++ infra/otel/grafana/dashboards/messenger.json | 174 ++++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/datasources.yml | 39 + infra/otel/loki.yml | 34 + infra/otel/prometheus.yml | 14 + infra/otel/tempo.yml | 27 + migrations/001_initial.sql | 43 + migrations/002_idempotency.sql | 16 + migrations/003_attachments.sql | 59 ++ migrations/004_pipeline_replay.sql | 16 + migrations/005_observability_context.sql | 3 + requirements.txt | 12 + scripts/bootstrap-secrets.py | 96 ++ scripts/bootstrap-secrets.sh | 3 + scripts/reconcile-loop.sh | 33 + scripts/restart.sh | 20 + scripts/stop-one.sh | 19 + src/__init__.py | 1 + src/app.py | 41 + src/bootstrap_s3.py | 47 + src/config.py | 61 ++ src/errors.py | 26 + src/handlers.py | 633 +++++++++++++ src/jobs.py | 50 ++ src/migrate.py | 24 + src/object_store.py | 76 ++ src/pipeline_versions.py | 4 + src/publisher.py | 101 +++ src/reconcile.py | 128 +++ src/replay_attachment.py | 34 + src/security.py | 53 ++ src/server.py | 13 + src/store.py | 848 ++++++++++++++++++ src/telemetry.py | 296 ++++++ src/worker.py | 383 ++++++++ student-tests/test_attachments.py | 98 ++ student-tests/test_idempotency_race.py | 47 + .../test_outbox_recovery_integration.py | 66 ++ student-tests/test_pipeline_faults.py | 485 ++++++++++ .../test_pipeline_replay_integration.py | 167 ++++ student-tests/test_reconcile.py | 102 +++ student-tests/test_reconcile_integration.py | 211 +++++ student-tests/test_worker.py | 45 + .../test_worker_recovery_integration.py | 78 ++ student-tests/track1_benchmark.py | 360 ++++++++ student-tests/upload_concurrency.py | 151 ++++ 60 files changed, 6317 insertions(+), 69 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/incidents.json create mode 100644 evidence/pipeline-faults.json create mode 100644 evidence/track1-observability-cost.json create mode 100644 haproxy.cfg create mode 100644 infra/otel/alerts.yml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/grafana/dashboards/messenger.json create mode 100644 infra/otel/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/otel/grafana/provisioning/datasources/datasources.yml create mode 100644 infra/otel/loki.yml create mode 100644 infra/otel/prometheus.yml create mode 100644 infra/otel/tempo.yml create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_pipeline_replay.sql create mode 100644 migrations/005_observability_context.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap-secrets.py create mode 100755 scripts/bootstrap-secrets.sh create mode 100755 scripts/reconcile-loop.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/config.py create mode 100644 src/errors.py create mode 100644 src/handlers.py create mode 100644 src/jobs.py create mode 100644 src/migrate.py create mode 100644 src/object_store.py create mode 100644 src/pipeline_versions.py create mode 100644 src/publisher.py create mode 100644 src/reconcile.py create mode 100644 src/replay_attachment.py create mode 100644 src/security.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 src/telemetry.py create mode 100644 src/worker.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_idempotency_race.py create mode 100644 student-tests/test_outbox_recovery_integration.py create mode 100644 student-tests/test_pipeline_faults.py create mode 100644 student-tests/test_pipeline_replay_integration.py create mode 100644 student-tests/test_reconcile.py create mode 100644 student-tests/test_reconcile_integration.py create mode 100644 student-tests/test_worker.py create mode 100644 student-tests/test_worker_recovery_integration.py create mode 100644 student-tests/track1_benchmark.py create mode 100644 student-tests/upload_concurrency.py diff --git a/.env.example b/.env.example index 2b32655..f507f83 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,18 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +API_CPU_LIMIT=1.0 +API_MEMORY_LIMIT=512M +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +MAX_CONCURRENT_UPLOADS=10 +WORKER_CONCURRENCY=1 +WORKER_REPLICAS=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +RECONCILE_INTERVAL_SECONDS=900 +RECONCILE_GRACE_SECONDS=3600 +GRAFANA_PORT=3000 +OTEL_SDK_DISABLED=false +OTEL_TRACES_SAMPLER=parentbased_traceidratio +OTEL_TRACES_SAMPLER_ARG=1.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..be99144 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +CMD ["python3", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d17..71e2884 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,12 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load benchmark-observability test-attachments test-idempotency-race test-reconcile \ + test-worker test-pipeline-faults test-pipeline-replay test-outbox-recovery test-worker-recovery \ + test-reconcile-integration test-upload-concurrency test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check/validate: offline contract checks; test: public smoke; student targets: focused attachment/pipeline checks" check: $(PYTHON) scripts/check_contract.py @@ -21,6 +23,39 @@ test: load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +benchmark-observability: + $(PYTHON) student-tests/track1_benchmark.py --base-url "$(BASE_URL)" + +test-attachments: + $(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency-race: + $(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + $(PYTHON) student-tests/test_reconcile.py + +test-worker: + $(PYTHON) student-tests/test_worker.py + +test-pipeline-faults: + $(PYTHON) student-tests/test_pipeline_faults.py + +test-pipeline-replay: + $(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-outbox-recovery: + $(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-worker-recovery: + $(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile-integration: + $(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-upload-concurrency: + $(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/README.md b/README.md index a607238..465d80d 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +В этой реализации сохранён сервер Lab 5 и добавлены OpenTelemetry-инструментация, Collector, Prometheus, Loki, Tempo, Grafana, alert rules и эксперимент по sampling. Язык и framework свободные; UI не обязателен. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) ## Откуда начинаем -Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. +Бизнес-API и фоновый pipeline перенесены из lab5; контракт не меняется. Стенд сохраняет API-реплики, PostgreSQL, S3, RabbitMQ, publisher, worker и reconciler. **Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. @@ -47,24 +47,25 @@ ## Как начать и проверить ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +# Локальный offline check, Python 3.9+: make check -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test -``` - -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. - -```sh -# Compose-вариант после реализации: +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +cp .env.example .env +scripts/bootstrap-secrets.sh docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +# Дождитесь health/ready. По умолчанию API публикуется на localhost:8080. +make test PYTHON=.venv/bin/python BASE_URL=http://localhost:8080 ``` +`.env.example` содержит локальные настройки без секретов. Секреты генерируются в `secrets/` и не должны коммититься. `compose.example.yaml` — worksheet с пустым `services`, реальный запускаемый стенд — `compose.yaml`. Образы закреплены версиями, данные приложения и трёх backend хранятся в named volumes. + +Для полной OpenAPI-валидации установите `requirements-dev.txt` и выполните `make validate PYTHON=.venv/bin/python`. Основные локальные проверки pipeline: `make test-worker`, `make test-reconcile`, `make test-pipeline-faults`. Сводный отчёт о затратах sampling запускается после старта стенда командой `make benchmark-observability PYTHON=.venv/bin/python BASE_URL=http://localhost:8080`. + +Grafana доступна на `http://localhost:3000`; пароль администратора находится в локальном `secrets/grafana_admin_password`. OTLP endpoints и telemetry backends доступны только внутри Compose-сети. При занятом порте задайте другой `APP_PORT` в локальном `.env` и соответствующий `BASE_URL`. + Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): diff --git a/REPORT.md b/REPORT.md index b97ae42..8c882d7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,74 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны. +- Целевая оценка: 5; трек 1 — цена наблюдаемости. +- Commit/PR: не создан. +- Источники и инструменты: README и CONTRIBUTING лабораторной 6; документация OpenTelemetry, Prometheus, Grafana, Loki и Tempo; исходники Lab 5; помощь ИИ использовалась для переноса, реализации и проверки. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose v2 и Python 3.9+ (проверено на Python 3.12.7). Для локального запуска: + +```sh +make check +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +cp .env.example .env +scripts/bootstrap-secrets.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться успешного ответа GET /health/ready. +make test PYTHON=.venv/bin/python BASE_URL=http://localhost:8080 +make validate PYTHON=.venv/bin/python +docker compose --env-file .env stop +``` + +При занятом порте 8080 задайте `APP_PORT` в локальном `.env` и используйте соответствующий `BASE_URL`; в проверенной среде использовался порт 18080. `scripts/bootstrap-secrets.sh` создаёт локальные секреты в `secrets/`; не копируйте их в Git. Обычная остановка сохраняет named volumes. Только для намеренного удаления учебных данных выполните `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Перенесены API и фоновый pipeline Lab 5 без изменения публичного контракта. API записывает бизнес-состояние и transactional outbox в PostgreSQL; publisher передаёт задания в RabbitMQ, worker обрабатывает вложения и сохраняет оригиналы/варианты в S3-совместимом SeaweedFS. PostgreSQL хранит авторитетное состояние и метаданные; reconciler сверяет их с объектами хранилища. Доставка at-least-once, повторные эффекты ограничены идемпотентностью и fencing. + +API, publisher и worker экспортируют traces, metrics и структурные logs через OTLP в Collector. W3C trace context сохраняется в outbox и переносится через broker. Prometheus собирает метрики Collector, Loki хранит логи, Tempo — traces, Grafana загружает datasource и dashboard из provisioning. Метрики используют route templates и другие низкокардинальные атрибуты, не идентификаторы пользователей, чатов или сообщений. В telemetry не включаются тела сообщений, credentials, filenames или signed URLs. + +SDK batch queues ограничены 4096 records, batch — 512; Collector ограничен memory limiter 384 MiB (spike 96 MiB), batch до 1024 элементов и очередями exporters по 256 requests с retry не более 30 секунд. Для локального инцидента с недоступным Tempo API smoke продолжил проходить, а измеренная память API и Collector оставалась около 57–76 MiB. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke и контракты | `make check`, `make validate`, `make test PYTHON=.venv/bin/python BASE_URL=http://localhost:18080` | Все проверки прошли; smoke прошёл с включённой телеметрией | README и `tests/` | +| Уровень 3: pipeline и API | `make test-attachments`, `make test-idempotency-race`, `make test-pipeline-replay` | Валидация вложений прошла; 12 конкурентных запросов с одним idempotency key дали одно сообщение; replay прошёл | `student-tests/`; `evidence/pipeline-faults.json` | +| Уровень 3: Grafana и сигналы | Grafana provisioning и Tempo trace | Загружен dashboard из 14 панелей и три datasource. Trace `69a718dd0f13278bddb76e719b8b23f1` связал upload/API, publisher, worker, S3 и DB | `infra/otel/grafana/`; Tempo | +| Уровень 4: восстановление данных | `make test-outbox-recovery`, `make test-worker-recovery`, `make test-reconcile-integration` | Outbox и worker восстановили вложение до `ready`; reconcile восстановил отсутствующий объект, удалил сироты и сохранил свежий объект до истечения grace period | `student-tests/` | +| Уровень 4: dashboard и SLO | Prometheus rules API, `promtool check rules`, Grafana dashboard | Доступны latency histogram, ошибки, очередь, DB pool и worker; rolling SLI `messenger:api_availability:ratio7d` вернул 1 при отсутствии 5xx. Правила прошли проверку; выражения нулевой ошибки и возраста очереди исправлены по результатам проверки | `infra/otel/alerts.yml`; `infra/otel/grafana/dashboards/messenger.json` | +| Уровень 4: отказ и alerts | Остановить worker с ожидающей вложением; затем восстановить. Отдельно остановить PostgreSQL и отправить 90 последовательных API-запросов | `MessengerQueueLag` перешёл firing и стал inactive после обработки задания. PostgreSQL outage дал 90 ответов 503; `MessengerApiErrorBudgetBurn` перешёл firing. После запуска PostgreSQL и API readiness восстановилась | `evidence/incidents.json` | +| Уровень 4: telemetry backend | Остановить Tempo на 40 секунд и выполнить `make test` | API smoke прошёл, память API/Collector осталась ограниченной; после возврата Tempo API readiness восстановилась | `evidence/incidents.json`; `infra/otel/collector.yaml` | +| Трек 1: цена наблюдаемости | `.venv/bin/python student-tests/track1_benchmark.py --duration 15 --concurrency 2 --target-rps 20 --warmup-seconds 3 --base-url http://localhost:18080` | Четыре режима завершили по 300 запросов без ошибок; результаты приведены ниже | `evidence/track1-observability-cost.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Дополнительно прошли `make test-reconcile`, `make test-worker`, `make test-pipeline-faults`, `make test-upload-concurrency`. Upload concurrency обработал 10 загрузок при 10 параллельных операциях; измеренный пик прироста working set API-контейнеров — около 14 MiB. -## Эксперименты и отказы +## Исследовательский трек 1 — цена наблюдаемости -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: sampling снижает число сохранённых spans и объём traces при сохранении метрик и логов. Нагрузка одинаковая для режимов: paced synthetic POST, target 20 RPS, concurrency 2, 15 секунд, прогрев 3 секунды; macOS 27 arm64, 10 логических CPU, Compose-контейнеры. Все режимы завершили 300 запросов с нулём ошибок; фактическая пропускная способность была 16.97–17.85 успешных RPS. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Режим | Traces / доля запросов | Успешный RPS | p50 / p95 / p99, ms | Средний CPU, сумма контейнеров | Пиковая память, MB | Tempo volume delta | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| SDK off | 0 / 0% | 17.750 | 7.449 / 12.198 / 14.054 | 17.18% | 331 | 120 KiB | +| Sampling 100% | 300 / 100% | 16.970 | 8.220 / 14.766 / 37.456 | 25.32% | 340 | 418 KiB | +| Sampling 25% | 88 / 29.3% | 17.845 | 6.822 / 12.894 / 14.415 | 26.23% | 352 | 3.90 MiB | +| Sampling 10% | 25 / 8.3% | 17.393 | 7.351 / 13.809 / 17.187 | 28.82% | 340 | 104 KiB | -## Ограничения и следующие шаги +В sampling-режимах в среднем было около 8 spans на message trace. Поэтому приблизительная оценка не записанных spans составляет 0 при 100%, 1696 при 25% и 2200 при 10%; это экстраполяция по первым 20 traces, а не счётчик доставки Collector. SDK off отключает telemetry SDK целиком, поэтому там ожидалось около 2400 spans, если бы трассировка была включена. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Для дальнейшего использования выбран sampling 25%: в этом прогоне он сохранил около 29% message traces и уменьшил оценочное число spans примерно на 71% относительно 100%, оставляя достаточно трасс для диагностики. + +## Инциденты и runbooks + +| Воздействие | Причина и наблюдение | Восстановление / данные | +| --- | --- | --- | +| Worker остановлен при одной queued attachment | API принял задание; рост возраста очереди виден в метрике, `MessengerQueueLag` сработал. | Worker запущен, задание обработано до `ready`, очередь осушена, alert inactive. Потери вложения нет. | +| PostgreSQL остановлен на время серии запросов | API вернул 503 на 90 последовательных запросов; `MessengerApiErrorBudgetBurn` сработал по метрике 5xx. | PostgreSQL запущен, API контейнеры перезапущены для сброса stale pool connections, `/health/ready` восстановился. Запросы получили 503; частично принятых операций не было. | +| Tempo недоступен 40 секунд | При остановленном trace backend API smoke оставался успешен; память API/Collector не демонстрировала неограниченного роста. | Tempo запущен, API readiness осталась healthy. | + +Для API 5xx: проверить SLI/alert, сгруппировать ошибки по route template, открыть trace и correlated log, проверить DB/S3 spans и состояние соответствующего backend; после восстановления подтвердить health/readiness и возврат error ratio. Для queue lag: проверить depth/age, worker consuming logs и broker; восстановить worker, дождаться `ready` и inactive alert. Для telemetry outage: проверить Collector exporter failures и backend health; бизнес-операции проверять отдельно от полноты telemetry. diff --git a/compose.example.yaml b/compose.example.yaml index dfa1cfb..ffce4c0 100644 --- a/compose.example.yaml +++ b/compose.example.yaml @@ -1,6 +1,5 @@ -# Topology worksheet, deliberately NOT a runnable stack (no application supplied). -# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. -# Service names below are logical roles, not imposed container names. +# Topology worksheet, deliberately NOT a runnable stack. +# Implemented roles are in compose.yaml; these names are logical, not container names. x-lab-number: 6 x-required-roles: - api @@ -10,8 +9,8 @@ x-required-roles: - broker - worker - otel_collector - - telemetry_storage + - prometheus + - loki + - tempo - grafana -# TODO: implement real images/builds, network wiring, volumes, healthchecks, -# credentials and bootstrap. Details: contracts/infrastructure.json and README.md. services: {} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ca73665 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,468 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + secrets: + - source: rabbitmq_config + target: /etc/rabbitmq/conf.d/10-defaults.conf + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + restart: unless-stopped + + s3-core: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - server + - -dir=/data + - -filer=true + - -s3=false + volumes: + - seaweed_data:/data + healthcheck: + test: + - CMD-SHELL + - "curl --fail --silent http://127.0.0.1:8888/ >/dev/null" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + networks: + - seaweed_internal + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - s3 + - -filer=s3-core:8888 + - -config=/run/secrets/s3_config + - -port=8333 + - -port.https=0 + - -port.grpc=0 + - -port.iceberg=0 + - -port.lance=0 + - -concurrentFileUploadLimit=10 + - -concurrentUploadLimitMB=100 + - -readerCacheSizeMB=64 + - -autoCreateBucket=false + - -ip.bind=0.0.0.0 + secrets: + - s3_config + depends_on: + s3-core: + condition: service_healthy + networks: + - default + - seaweed_internal + healthcheck: + test: + - CMD-SHELL + - "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3-bootstrap: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/bootstrap_s3.py"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10} + UPLOAD_TIMEOUT_SECONDS: "60" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX_SIZE: "8" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + DB_LOCK_TIMEOUT_MS: "2000" + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: + - CMD + - python3 + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" + interval: 3s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEMORY_LIMIT:-512M}" + stop_grace_period: 20s + restart: unless-stopped + + publisher: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.publisher"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-publisher + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + MAX_IMAGE_PIXELS: "40000000" + DB_POOL_MAX_SIZE: "2" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + otel-collector: + condition: service_started + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + reconciler: + build: + context: . + dockerfile: Dockerfile + command: ["sh", "scripts/reconcile-loop.sh"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.163.2 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + - tempo + - loki + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + prometheus: + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + volumes: + - ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + depends_on: + - otel-collector + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2 + command: ["-config.file=/etc/loki/loki.yml"] + volumes: + - ./infra/otel/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.1 + command: ["-config.file=/etc/tempo/tempo.yml"] + volumes: + - ./infra/otel/tempo.yml:/etc/tempo/tempo.yml:ro + - tempo_data:/var/tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + grafana: + image: grafana/grafana:11.4.0 + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + secrets: + - grafana_admin_password + volumes: + - grafana_data:/var/lib/grafana + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + - tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.7-alpine + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_config: + file: ./secrets/s3.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + rabbitmq_password: + file: ./secrets/rabbitmq_password + rabbitmq_config: + file: ./secrets/rabbitmq.conf + broker_url: + file: ./secrets/broker_url + grafana_admin_password: + file: ./secrets/grafana_admin_password + +volumes: + postgres_data: + seaweed_data: + rabbitmq_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + +networks: + seaweed_internal: + internal: true diff --git a/evidence/incidents.json b/evidence/incidents.json new file mode 100644 index 0000000..95524c7 --- /dev/null +++ b/evidence/incidents.json @@ -0,0 +1,39 @@ +{ + "result": "PASS", + "scenarios": [ + { + "scenario": "worker_outage", + "injected_fault": "Stopped the worker while one attachment job was queued.", + "alert": "MessengerQueueLag", + "alert_fired": true, + "alert_resolved_after_recovery": true, + "job_reached_ready": true, + "data_loss": false + }, + { + "scenario": "postgres_outage", + "injected_fault": "Stopped PostgreSQL during 90 sequential API user-creation requests.", + "api_responses": { + "503": 90 + }, + "alert": "MessengerApiErrorBudgetBurn", + "alert_fired": true, + "api_ready_after_database_and_api_recovery": true + }, + { + "scenario": "tempo_outage", + "injected_fault": "Stopped Tempo for 40 seconds.", + "public_smoke_passed_during_outage": true, + "api_and_collector_memory_mib_while_stopped": { + "api_replica_1": 58.6, + "api_replica_2": 58.6, + "otel_collector": 76.2 + }, + "api_ready_after_recovery": true + } + ], + "limitations": [ + "The fault checks are local single-run experiments, not production availability tests.", + "Telemetry backends may drop data after bounded exporter retries during an outage." + ] +} diff --git a/evidence/pipeline-faults.json b/evidence/pipeline-faults.json new file mode 100644 index 0000000..28b2b2b --- /dev/null +++ b/evidence/pipeline-faults.json @@ -0,0 +1,58 @@ +{ + "fixture": "tests/fixtures/sample.png", + "invariants": { + "ambiguous_confirm_republishes_stable_job": true, + "confirmed_work_eventually_ready": true, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + "duplicate_effects_create_no_second_variant_set": true, + "stale_attempt_writes_no_effects": true, + "stale_pipeline_cannot_publish_current_metadata": true + }, + "result": "PASS", + "scenarios": [ + { + "deliveries": 2, + "duplicate_delivery_writes": 0, + "effective_object_writes": 4, + "scenario": "duplicate_delivery_after_commit_before_ack" + }, + { + "attempts": 2, + "scenario": "crash_after_claim", + "terminal_status": "ready" + }, + { + "attempts": 2, + "scenario": "crash_after_original_put", + "terminal_status": "ready" + }, + { + "current_attempt": 2, + "scenario": "out_of_order_attempt_delivery", + "stale_attempt_extra_writes": 0 + }, + { + "current_pipeline_version": 2, + "published_variant_versions": [ + "v2" + ], + "scenario": "stale_pipeline_after_newer_version" + }, + { + "outbox_recovered_after_restart": true, + "published_after_restart": 1, + "scenario": "publisher_crash_before_publish" + }, + { + "deliveries": 2, + "same_message_id": true, + "scenario": "publish_confirm_lost_after_broker_accept" + }, + { + "configured_timeout_seconds": 1, + "observed_seconds": 1.006, + "scenario": "transform_timeout_terminated" + } + ], + "track": "2" +} diff --git a/evidence/track1-observability-cost.json b/evidence/track1-observability-cost.json new file mode 100644 index 0000000..8a14325 --- /dev/null +++ b/evidence/track1-observability-cost.json @@ -0,0 +1,208 @@ +{ + "experiment": "telemetry overhead under fixed-rate synthetic POST load", + "hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.", + "method": { + "duration_seconds": 15, + "concurrency": 2, + "target_rps": 20, + "warmup_seconds": 3, + "telemetry_flush_wait_seconds": 15, + "hardware": { + "platform": "macOS-27.0.1-arm64-arm-64bit-Mach-O", + "processor": "arm", + "logical_cpus": 10 + }, + "sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.", + "storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size." + }, + "limitations": [ + "The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.", + "Database history grows between modes; all modes use identical duration and concurrency.", + "Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.", + "Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.", + "CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes." + ], + "results": [ + { + "configuration": "off", + "sdk_disabled": true, + "trace_sampling_ratio": 1.0, + "load": { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": 15, + "target_rps": 20, + "elapsed_seconds": 16.901, + "concurrency": 2, + "successful_requests": 300, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 17.75, + "successful_latency_ms": { + "p50": 7.449, + "p95": 12.198, + "p99": 14.054 + }, + "docker_resource_samples": 4, + "docker_resources": { + "mean_cpu_percent_sum": 17.18, + "peak_cpu_percent_sum": 46.29, + "peak_memory_bytes_sum": 331140298, + "services": [ + "api", + "worker", + "publisher", + "otel-collector" + ] + } + }, + "window_unix_seconds": { + "start": 1793781279, + "end": 1793781296 + }, + "sampled_message_trace_count": 0, + "estimated_spans_per_message_trace": 0.0, + "persistent_storage_bytes_delta": { + "prometheus": 25354, + "loki": 0, + "tempo": 122589 + }, + "estimated_spans_not_recorded": 2400.0 + }, + { + "configuration": "sample-100-percent", + "sdk_disabled": false, + "trace_sampling_ratio": 1.0, + "load": { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": 15, + "target_rps": 20, + "elapsed_seconds": 17.678, + "concurrency": 2, + "successful_requests": 300, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 16.97, + "successful_latency_ms": { + "p50": 8.22, + "p95": 14.766, + "p99": 37.456 + }, + "docker_resource_samples": 4, + "docker_resources": { + "mean_cpu_percent_sum": 25.32, + "peak_cpu_percent_sum": 49.25, + "peak_memory_bytes_sum": 340199994, + "services": [ + "api", + "worker", + "publisher", + "otel-collector" + ] + } + }, + "window_unix_seconds": { + "start": 1793781337, + "end": 1793781354 + }, + "sampled_message_trace_count": 300, + "estimated_spans_per_message_trace": 8.0, + "persistent_storage_bytes_delta": { + "prometheus": 27422, + "loki": 240231, + "tempo": 428182 + }, + "estimated_spans_not_recorded": 0.0 + }, + { + "configuration": "sample-25-percent", + "sdk_disabled": false, + "trace_sampling_ratio": 0.25, + "load": { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": 15, + "target_rps": 20, + "elapsed_seconds": 16.811, + "concurrency": 2, + "successful_requests": 300, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 17.845, + "successful_latency_ms": { + "p50": 6.822, + "p95": 12.894, + "p99": 14.415 + }, + "docker_resource_samples": 4, + "docker_resources": { + "mean_cpu_percent_sum": 26.23, + "peak_cpu_percent_sum": 47.71, + "peak_memory_bytes_sum": 352195705, + "services": [ + "api", + "worker", + "publisher", + "otel-collector" + ] + } + }, + "window_unix_seconds": { + "start": 1793781394, + "end": 1793781411 + }, + "sampled_message_trace_count": 88, + "estimated_spans_per_message_trace": 8.0, + "persistent_storage_bytes_delta": { + "prometheus": 27362, + "loki": -7107212, + "tempo": 4092487 + }, + "estimated_spans_not_recorded": 1696.0 + }, + { + "configuration": "sample-10-percent", + "sdk_disabled": false, + "trace_sampling_ratio": 0.1, + "load": { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": 15, + "target_rps": 20, + "elapsed_seconds": 17.248, + "concurrency": 2, + "successful_requests": 300, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 17.393, + "successful_latency_ms": { + "p50": 7.351, + "p95": 13.809, + "p99": 17.187 + }, + "docker_resource_samples": 4, + "docker_resources": { + "mean_cpu_percent_sum": 28.82, + "peak_cpu_percent_sum": 49.75, + "peak_memory_bytes_sum": 339979794, + "services": [ + "api", + "worker", + "publisher", + "otel-collector" + ] + } + }, + "window_unix_seconds": { + "start": 1793781454, + "end": 1793781471 + }, + "sampled_message_trace_count": 25, + "estimated_spans_per_message_trace": 8.0, + "persistent_storage_bytes_delta": { + "prometheus": 31886, + "loki": 239322, + "tempo": 106458 + }, + "estimated_spans_not_recorded": 2200.0 + } + ], + "balanced_configuration": "sample-25-percent" +} diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 0000000..9dedf30 --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + option http-keep-alive + timeout connect 2s + timeout client 15s + timeout server 15s + timeout http-request 5s + timeout check 1s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend messenger + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + option redispatch + default-server inter 1s fall 2 rise 2 + server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..386510d 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,9 @@ -# Конфигурация наблюдаемости — реализовать +# OpenTelemetry deployment -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +The Compose stack pins Collector Contrib 0.163.2, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.7.1, and Grafana 11.4.0. OTLP gRPC/HTTP listeners and backend query ports are private to the Compose network; only the API load balancer and Grafana UI bind to loopback. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +`collector.yaml` receives all three OTLP signals. Metrics are exposed to Prometheus on the private Collector endpoint; logs are exported to Loki's OTLP receiver and traces to Tempo. Memory limiter and batch processors bound Collector memory. Trace/log exporter queues are capped at 256 requests and retry for at most 30 seconds; SDK span/log queues are capped at 4096 records with batches of at most 512. On backend outage, telemetry is dropped after bounded retries rather than blocking API work indefinitely. Metrics are pull-exported and can be missed while Prometheus is unavailable. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Loki, Tempo, and Prometheus retain local data in named volumes. Grafana datasources, the dashboard, and Prometheus alert rules are provisioned from this directory. Prometheus evaluates the 7-day rolling API availability SLI and local alerts; use the alert annotations and `REPORT.md` runbooks to investigate. + +API, publisher, and worker share a W3C trace context through the database outbox and the broker payload. Application logs contain only structured metadata, trace/span IDs, request IDs, route templates, and sanitized event names. Message bodies, credentials, filenames, and signed URLs are not exported. diff --git a/infra/otel/alerts.yml b/infra/otel/alerts.yml new file mode 100644 index 0000000..50e245a --- /dev/null +++ b/infra/otel/alerts.yml @@ -0,0 +1,42 @@ +groups: + - name: messenger-slo + interval: 10s + rules: + - record: messenger:api_5xx_ratio:rate5m + expr: | + (sum(rate(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0)) + / + clamp_min(sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])), 1e-9) + - record: messenger:api_availability:ratio7d + expr: | + 1 - + (sum(increase(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[7d])) or vector(0)) + / + clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[7d])), 1) + - alert: MessengerApiErrorBudgetBurn + expr: messenger:api_5xx_ratio:rate5m > 0.01 + for: 2m + labels: + severity: warning + annotations: + summary: API error budget burn exceeds the 1% short-window budget + description: The rolling 5-minute business API 5xx ratio is above 1%. + runbook: Check API traces for the failing route, then inspect correlated logs and DB/S3 spans. + - alert: MessengerQueueLag + expr: max(messenger_queue_oldest_age_seconds) > 60 and max(messenger_queue_depth) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Oldest attachment job has waited for more than 60 seconds + description: Inspect worker availability, RabbitMQ health, and the worker trace for the oldest job. + runbook: Check queue depth and age, worker consuming logs, broker, then recover the worker and confirm resolved state. + - alert: MessengerDatabasePoolSaturation + expr: max(messenger_db_pool_requests_waiting) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Requests are waiting for a database connection + description: Compare DB waiters with request traces and PostgreSQL health. + runbook: Inspect DB spans and pool use; verify database health and connection limits. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e81bc0a --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,66 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 384 + spike_limit_mib: 96 + batch: + timeout: 1s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + send_timestamps: true + resource_to_telemetry_conversion: + enabled: true + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + telemetry: + logs: + level: warn + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json new file mode 100644 index 0000000..04e026b --- /dev/null +++ b/infra/otel/grafana/dashboards/messenger.json @@ -0,0 +1,174 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": {"type": "grafana", "uid": "-- Grafana --"}, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 0}, + "id": 1, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_route}}", "refId": "A"}], + "title": "API requests per second", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 12, "y": 0}, + "id": 2, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",http_route!~\"/health/.*\"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m])), 1e-9)", "legendFormat": "5xx / business requests", "refId": "A"}], + "title": "Business API 5xx ratio (5m)", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 7}, + "id": 3, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "API latency percentiles from merged histogram buckets", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 7}, + "id": 4, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "Worker job duration percentiles from histogram buckets", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 15}, + "id": 5, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[5m]))", "legendFormat": "{{outcome}}", "refId": "A"}], + "title": "Jobs processed / failed per second", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 15}, + "id": 6, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_queue_depth)", "legendFormat": "queued attachments", "refId": "A"}], + "title": "Durable attachment backlog", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 15}, + "id": 7, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest queued job", "refId": "A"}], + "title": "Age of oldest queued attachment", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 22}, + "id": 8, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_db_pool_connections_used)", "legendFormat": "used", "refId": "A"}, {"expr": "max(messenger_db_pool_requests_waiting)", "legendFormat": "waiting", "refId": "B"}], + "title": "Database pool use / waiters", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 22}, + "id": 9, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "busy / capacity", "refId": "A"}], + "title": "Worker saturation", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 22}, + "id": 10, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_process_memory_usage)", "legendFormat": "RSS", "refId": "A"}], + "title": "Process memory", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 29}, + "id": 11, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_process_cpu_utilization)", "legendFormat": "CPU / core", "refId": "A"}], + "title": "Process CPU utilization", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 7, "w": 16, "x": 8, "y": 29}, + "id": 12, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachments_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95 time to ready", "refId": "A"}], + "title": "Durable acceptance to attachment ready", + "type": "timeseries" + }, + { + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 36}, + "id": 13, + "options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": true, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true}, + "targets": [{"expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", "refId": "A"}], + "title": "Structured API / worker logs", + "type": "logs" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0.9, "max": 1}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 44}, + "id": 14, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}}, + "targets": [{"expr": "messenger:api_availability:ratio7d", "legendFormat": "7-day availability; objective 99%", "refId": "A"}], + "title": "API SLI: 7-day rolling availability (99% SLO)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["messenger", "observability"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timezone": "browser", + "title": "Messenger observability", + "uid": "messenger-observability", + "version": 1 +} diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yml b/infra/otel/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..4db9d1f --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: Messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: true + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yml b/infra/otel/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..be6554b --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,39 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([a-f0-9]+)"' + url: "$${__value.raw}" + datasourceUid: tempo + editable: false + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus + editable: false diff --git a/infra/otel/loki.yml b/infra/otel/loki.yml new file mode 100644 index 0000000..5c522b8 --- /dev/null +++ b/infra/otel/loki.yml @@ -0,0 +1,34 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/otel/prometheus.yml b/infra/otel/prometheus.yml new file mode 100644 index 0000000..7a4ea69 --- /dev/null +++ b/infra/otel/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:9464"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/infra/otel/tempo.yml b/infra/otel/tempo.yml new file mode 100644 index 0000000..19460c2 --- /dev/null +++ b/infra/otel/tempo.yml @@ -0,0 +1,27 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + trace_idle_period: 10s + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..f5431a5 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..aae5c1a --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idempotency_key), + CONSTRAINT idempotency_method_post CHECK (method = 'POST'), + CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..a4178a7 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,59 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank; + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + staging_key text NOT NULL UNIQUE, + job_id uuid NOT NULL UNIQUE, + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + lease_until timestamptz, + status varchar(16) NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'), + error_code varchar(64), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + attempt integer NOT NULL CHECK (attempt >= 1), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + UNIQUE (job_id, attempt) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, created_at) + WHERE published_at IS NULL; + +CREATE TABLE reconciliation_incidents ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')), + first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, incident_type) +); diff --git a/migrations/004_pipeline_replay.sql b/migrations/004_pipeline_replay.sql new file mode 100644 index 0000000..424dcc5 --- /dev/null +++ b/migrations/004_pipeline_replay.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachment_outbox + DROP CONSTRAINT attachment_outbox_job_id_fkey; + +CREATE TABLE attachment_pipeline_events ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + prior_job_id uuid NOT NULL, + job_id uuid NOT NULL, + prior_version integer NOT NULL CHECK (prior_version >= 1), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + action varchar(16) NOT NULL CHECK (action = 'replay'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_pipeline_events_attachment_created_idx + ON attachment_pipeline_events (attachment_id, created_at); diff --git a/migrations/005_observability_context.sql b/migrations/005_observability_context.sql new file mode 100644 index 0000000..bd52f66 --- /dev/null +++ b/migrations/005_observability_context.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_outbox + ADD COLUMN traceparent varchar(55), + ADD COLUMN tracestate varchar(512); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..45ee36a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 +boto3==1.35.99 +Pillow==11.0.0 +pika==1.3.2 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +opentelemetry-instrumentation-psycopg==0.50b0 +opentelemetry-instrumentation-botocore==0.50b0 +psutil==6.1.1 diff --git a/scripts/bootstrap-secrets.py b/scripts/bootstrap-secrets.py new file mode 100644 index 0000000..f5901bc --- /dev/null +++ b/scripts/bootstrap-secrets.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import json +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" + + +def main(): + SECRET_DIR.mkdir(mode=0o700, exist_ok=True) + names = ( + "postgres_password", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + "s3_admin_access_key_id", + "s3_admin_secret_access_key", + "rabbitmq_password", + "rabbitmq.conf", + "broker_url", + "s3.json", + "grafana_admin_password", + ) + existing = [name for name in names if (SECRET_DIR / name).exists()] + if existing: + raise SystemExit( + "Secret files already exist; refusing to rotate credentials without a migration: " + + ", ".join(existing) + ) + + db_password = secrets.token_hex(24) + broker_password = secrets.token_urlsafe(32) + values = { + "postgres_password": db_password, + "database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger", + "s3_access_key_id": "mngr" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(40), + "s3_admin_access_key_id": "admin" + secrets.token_hex(7), + "s3_admin_secret_access_key": secrets.token_urlsafe(40), + "rabbitmq_password": broker_password, + "broker_url": f"amqp://messenger:{broker_password}@rabbitmq:5672/%2f", + "grafana_admin_password": secrets.token_urlsafe(32), + } + for name, value in values.items(): + path = SECRET_DIR / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + descriptor = os.open( + SECRET_DIR / "rabbitmq.conf", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write("default_user = messenger\n") + output.write(f"default_pass = {broker_password}\n") + s3_config = { + "identities": [ + { + "name": "bootstrap-admin", + "credentials": [ + { + "accessKey": values["s3_admin_access_key_id"], + "secretKey": values["s3_admin_secret_access_key"], + } + ], + "actions": ["Admin"], + }, + { + "name": "messenger-api", + "credentials": [ + { + "accessKey": values["s3_access_key_id"], + "secretKey": values["s3_secret_access_key"], + } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments", + ], + }, + ] + } + descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(s3_config, output, separators=(",", ":")) + output.write("\n") + print("Local secrets generated with owner-only permissions; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100755 index 0000000..5ea636b --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 "$(dirname "$0")/bootstrap-secrets.py" diff --git a/scripts/reconcile-loop.sh b/scripts/reconcile-loop.sh new file mode 100755 index 0000000..cbd430f --- /dev/null +++ b/scripts/reconcile-loop.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +interval=${RECONCILE_INTERVAL_SECONDS:-900} +grace=${RECONCILE_GRACE_SECONDS:-3600} + +case "$interval" in + ''|*[!0-9]*) + echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +case "$grace" in + ''|*[!0-9]*) + echo "RECONCILE_GRACE_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +if [ "$interval" -lt 60 ]; then + echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2 + exit 2 +fi +if [ "$grace" -lt 60 ]; then + echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2 + exit 2 +fi + +while : +do + python3 src/reconcile.py --grace-seconds "$grace" --apply + sleep "$interval" +done diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..3baf3e3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres s3-core rabbitmq prometheus loki tempo +docker compose up -d --wait postgres s3-core rabbitmq prometheus loki tempo +docker compose restart s3 +docker compose up -d --wait s3 +attempt=0 +until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 4 ]; then + echo "S3 object reads did not recover after restart" >&2 + exit 1 + fi + sleep 2 +done +docker compose restart api +docker compose restart publisher worker +docker compose restart otel-collector grafana +docker compose up -d --wait api publisher worker otel-collector grafana diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..59e50fe --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +found= +for container_id in $(docker compose ps --quiet --status running api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + found=$container_id + break + fi +done + +if [ -z "$found" ]; then + printf '%s\n' 'No running API container matched the requested instance alias.' >&2 + exit 1 +fi +docker stop --time 5 "$found" >/dev/null diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..5bc8da3 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..01f646a --- /dev/null +++ b/src/app.py @@ -0,0 +1,41 @@ +import logging +import os +import signal +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .store import Store +from .telemetry import initialize, start_runtime_metrics + +LOGGER = logging.getLogger("messenger") + + +def run_server(): + store = Store() + store.open() + start_runtime_metrics(store) + try: + handler = partial(MessengerHandler, store=store) + server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-api")) + run_server() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 0000000..3c3bc7a --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import logging +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger("messenger.s3.bootstrap") + + +def secret(name: str) -> str: + direct = os.getenv(name) + if direct: + return direct + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError(f"Set {name} or {name}_FILE") + + +def main(): + client = boto3.client( + "s3", + endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name="us-east-1", + aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"), + aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"), + config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}), + ) + bucket = os.getenv("S3_BUCKET", "messenger-attachments") + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + client.create_bucket(Bucket=bucket) + client.put_bucket_acl(Bucket=bucket, ACL="private") + LOGGER.info("private S3 bucket bootstrap completed") + + +if __name__ == "__main__": + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + main() diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..2dadff5 --- /dev/null +++ b/src/config.py @@ -0,0 +1,61 @@ +import os +import re +import socket +import uuid +from pathlib import Path + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10")) +UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60")) +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") + + +def secret_value(name: str, default: str = "") -> str: + value = os.getenv(name) + if value: + return value + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + if default: + return default + raise RuntimeError(f"Set {name} or {name}_FILE") + + +S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY") +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if IDEMPOTENCY_TTL_SECONDS < 86400: + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") +if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024: + raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB") +if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000: + raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million") +if MAX_CONCURRENT_UPLOADS < 1: + raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive") +if UPLOAD_TIMEOUT_SECONDS < 1: + raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..de78501 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,26 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass + + +class AttachmentNotReadyError(Exception): + pass + + +class ObjectMissingError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 0000000..88ab6f1 --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,633 @@ +import base64 +import hashlib +import json +import logging +import threading +import time +import tempfile +import uuid +import warnings +import zlib +from http.server import BaseHTTPRequestHandler +from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit +import unicodedata + +import psycopg +from opentelemetry import propagate +from opentelemetry.instrumentation.utils import suppress_instrumentation +from opentelemetry.trace import SpanKind, Status, StatusCode +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +from psycopg_pool import PoolTimeout + +from .config import ( + IDEMPOTENCY_KEY_RE, + INSTANCE_ID, + MAX_CONCURRENT_UPLOADS, + MAX_IMAGE_PIXELS, + MAX_INFLIGHT_REQUESTS, + MAX_REQUEST_BODY_BYTES, + MAX_UPLOAD_BYTES, + UPLOAD_TIMEOUT_SECONDS, + USER_RE, +) +from .errors import ( + ApiError, + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .security import hash_password, isoformat, valid_uuid +from .store import Store +from .telemetry import REQUEST_ID, record_http_request, tracer + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() +UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS) +IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1) + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def setup(self): + super().setup() + self.connection.settimeout(15) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + path = urlsplit(self.path).path + route = self._route_template(path) + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + parent_context = propagate.extract({key.lower(): value for key, value in self.headers.items()}) + started = time.perf_counter() + token = REQUEST_ID.set(request_id) + self._response_status = 500 + if route.startswith("/health/"): + try: + self._dispatch_request(method, request_id) + finally: + record_http_request(method, route, self._response_status, time.perf_counter() - started) + REQUEST_ID.reset(token) + return + with tracer().start_as_current_span( + f"{method} {route}", + context=parent_context, + kind=SpanKind.SERVER, + attributes={ + "http.request.method": method, + "http.route": route, + "http.request_id": request_id, + }, + ) as span: + try: + self._dispatch_request(method, request_id) + finally: + duration = time.perf_counter() - started + span.set_attribute("http.response.status_code", self._response_status) + if self._response_status >= 500: + span.set_status(Status(StatusCode.ERROR)) + record_http_request(method, route, self._response_status, duration) + REQUEST_ID.reset(token) + + @staticmethod + def _route_template(path: str) -> str: + if path in ("/health/live", "/health/ready", "/api/v1/users", "/api/v1/auth/sessions", + "/api/v1/auth/sessions/current", "/api/v1/users/me", "/api/v1/chats"): + return path + if path.startswith("/api/v1/chats/") and path.endswith("/attachments"): + return "/api/v1/chats/{chat_id}/attachments" + if path.startswith("/api/v1/chats/") and path.endswith("/messages"): + return "/api/v1/chats/{chat_id}/messages" + if path.startswith("/api/v1/chats/"): + return "/api/v1/chats/{chat_id}" + if path.startswith("/api/v1/attachments/") and path.endswith("/content"): + return "/api/v1/attachments/{attachment_id}/content" + if path.startswith("/api/v1/attachments/"): + return "/api/v1/attachments/{attachment_id}" + return "unmatched" + + def _dispatch_request(self, method: str, request_id: str): + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + with suppress_instrumentation(): + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "A required storage service is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST": + chat_id = path.rsplit("/", 2)[1] + self._handle_upload_attachment(chat_id, query, request_id) + elif path.startswith("/api/v1/attachments/") and method == "GET": + attachment_path = path.removeprefix("/api/v1/attachments/") + if attachment_path.endswith("/content"): + attachment_id = attachment_path.removesuffix("/content") + self._handle_download_attachment(attachment_id, query, request_id) + else: + self._handle_get_attachment(attachment_path, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except IdempotencyConflictError: + self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id) + except AttachmentNotReadyError: + self._send_error(409, "conflict", "Attachment is not ready", request_id) + except ObjectMissingError: + LOGGER.error("attachment object missing for request %s", request_id) + self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id) + except (BotoCoreError, ClientError): + LOGGER.warning("object storage unavailable for request %s", request_id) + self._send_error(503, "unavailable", "Object storage is unavailable", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + self._set_response_status(499) + self.close_connection = True + LOGGER.info("client disconnected before response for request %s", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + @staticmethod + def _trace_context(): + carrier = {} + propagate.inject(carrier) + return {"traceparent": carrier.get("traceparent"), "tracestate": carrier.get("tracestate")} + + def _set_response_status(self, status: int) -> None: + self._response_status = status + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self._set_response_status(204) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + idempotency_key = self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body: + raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids") + text = body["text"] + attachment_ids = body.get("attachment_ids", []) + if ( + not isinstance(text, str) + or len(text) > 4000 + or not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + or (not text.strip() and not attachment_ids) + ): + raise ApiError(400, "invalid_request", "Invalid message text") + attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids] + canonical_body = json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + message = self.store.create_idempotent_message( + chat_id, + actor["id"], + text, + attachment_ids, + idempotency_key, + canonical_body, + ) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]: + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + length = self.headers.get("Content-Length") + if length is None or self.headers.get("Transfer-Encoding"): + raise ApiError(400, "invalid_request", "A bounded Content-Length is required") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size <= 0: + raise ApiError(415, "unsupported_media_type", "Image is empty") + if size > MAX_UPLOAD_BYTES: + self.close_connection = True + remaining = min(size, MAX_UPLOAD_BYTES + 1) + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + break + self.connection.settimeout(min(15, remaining_time)) + try: + chunk = self.rfile.read(min(64 * 1024, remaining)) + except (TimeoutError, OSError): + LOGGER.info("stopped bounded drain of oversized upload") + break + if not chunk: + break + remaining -= len(chunk) + raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit") + content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + digest = hashlib.sha256() + bytes_read = 0 + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + try: + while bytes_read < size: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise ApiError(408, "invalid_request", "Image upload timed out") + self.connection.settimeout(min(15, remaining_time)) + chunk = self.rfile.read(min(64 * 1024, size - bytes_read)) + if not chunk: + raise ApiError(400, "invalid_request", "Incomplete image body") + content.write(chunk) + digest.update(chunk) + bytes_read += len(chunk) + except (TimeoutError, OSError): + content.close() + raise ApiError(400, "invalid_request", "Incomplete image body") from None + + try: + content.seek(0) + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + if actual_type is None or actual_type != declared_type: + raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type") + width, height = image.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") + image.verify() + content.seek(0) + with IMAGE_DECODE_GUARD: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + image.load() + content.seek(0) + return content, actual_type, size, digest.hexdigest() + except ApiError: + content.close() + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + content.close() + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error): + content.close() + raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None + + @staticmethod + def _valid_filename(filename: str) -> bool: + return ( + 1 <= len(filename) <= 255 + and not any(char in ("/", "\\", "\x00") for char in filename) + and not any(unicodedata.category(char) == "Cc" for char in filename) + ) + + def _handle_upload_attachment( + self, chat_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + if not self.store.can_access_chat(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Chat not found") + filenames = query.get("filename", []) + if len(filenames) != 1 or not self._valid_filename(filenames[0]): + raise ApiError(400, "invalid_request", "Invalid filename") + if not UPLOAD_GUARD.acquire(blocking=False): + raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted") + try: + content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type()) + try: + attachment = self.store.create_attachment( + chat_id, + actor["id"], + filenames[0], + content_type, + content, + size, + digest, + **self._trace_context(), + ) + finally: + content.close() + finally: + UPLOAD_GUARD.release() + if attachment is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(202, attachment, request_id) + + def _handle_get_attachment(self, attachment_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"]) + if attachment is None: + raise ApiError(404, "not_found", "Attachment not found") + self._send_json(200, attachment, request_id) + + def _handle_download_attachment( + self, attachment_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + variants = query.get("variant", ["original"]) + if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"): + raise ApiError(400, "invalid_request", "Invalid attachment variant") + variant = variants[0] + if variant != "original" and variant not in ("thumbnail", "crop", "optimized"): + raise ApiError(404, "not_found", "Attachment variant not found") + result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant) + if result is None: + raise ApiError(404, "not_found", "Attachment not found") + content_type, content = result + self._set_response_status(200) + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(content))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Cache-Control", "private, no-store") + self.end_headers() + self.wfile.write(content) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + self._set_response_status(status) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..b494c64 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,50 @@ +import json +import os +from pathlib import Path + +import pika + +QUEUE_NAME = "attachments.process.v1" +DEAD_QUEUE_NAME = "attachments.process.dead" +LEASE_NETWORK_BUDGET_SECONDS = 150 + + +def broker_url() -> str: + value = os.getenv("BROKER_URL") + if value: + return value + file_path = os.getenv("BROKER_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set BROKER_URL or BROKER_URL_FILE") + + +def connect() -> pika.BlockingConnection: + parameters = pika.URLParameters(broker_url()) + parameters.connection_attempts = 3 + parameters.retry_delay = 1 + parameters.socket_timeout = 5 + parameters.blocked_connection_timeout = 10 + return pika.BlockingConnection(parameters) + + +def declare_queues(channel: pika.adapters.blocking_connection.BlockingChannel) -> None: + channel.queue_declare( + queue=DEAD_QUEUE_NAME, + durable=True, + arguments={"x-queue-type": "quorum", "x-quorum-initial-group-size": 1}, + ) + channel.queue_declare( + queue=QUEUE_NAME, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-quorum-initial-group-size": 1, + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": DEAD_QUEUE_NAME, + }, + ) + + +def encode_job(job: dict) -> bytes: + return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..1ae2700 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 0000000..b49c67f --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,76 @@ +import logging +from datetime import datetime +from typing import BinaryIO, Iterator, Optional, Tuple, Union + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY +from .errors import ObjectMissingError + +LOGGER = logging.getLogger("messenger.s3") + + +class ObjectStore: + def __init__(self): + self.client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + + def put( + self, + key: str, + content: Union[bytes, BinaryIO], + content_type: str, + size_bytes: Optional[int] = None, + ) -> None: + if size_bytes is None: + size_bytes = len(content) + self.client.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=content, + ContentLength=size_bytes, + ContentType=content_type, + ) + + def get(self, key: str, max_bytes: int) -> bytes: + try: + response = self.client.get_object(Bucket=S3_BUCKET, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("NoSuchKey", "404", "NotFound"): + raise ObjectMissingError from exc + raise + body = response["Body"] + try: + value = body.read(max_bytes + 1) + finally: + body.close() + if len(value) > max_bytes: + raise RuntimeError("Stored object exceeds configured safety limit") + return value + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=S3_BUCKET, Key=key) + + def ready(self) -> bool: + self.client.head_bucket(Bucket=S3_BUCKET) + return True + + def objects(self) -> Iterator[Tuple[str, datetime]]: + pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET) + for page in pages: + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] diff --git a/src/pipeline_versions.py b/src/pipeline_versions.py new file mode 100644 index 0000000..5b87cfa --- /dev/null +++ b/src/pipeline_versions.py @@ -0,0 +1,4 @@ +PIPELINE_QUALITY = { + 1: {"jpeg": 85, "webp": 80}, + 2: {"jpeg": 80, "webp": 75}, +} diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 0000000..d2d8354 --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import logging +import os +import time +import uuid + +import pika +import psycopg +from opentelemetry import propagate +from opentelemetry.instrumentation.utils import suppress_instrumentation +from opentelemetry.trace import SpanKind + +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues, encode_job +from .security import isoformat +from .store import Store +from .telemetry import initialize, tracer + +LOGGER = logging.getLogger("messenger.publisher") + + +def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool: + with store.pool.connection() as conn: + with conn.transaction(): + with suppress_instrumentation(): + row = store.claim_outbox(conn) + if row is None: + return False + payload = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": isoformat(row["created_at"]), + } + parent_context = propagate.extract( + { + key: row.get(key) + for key in ("traceparent", "tracestate") + if row.get(key) is not None + } + ) + with tracer().start_as_current_span( + "attachment.publish", + context=parent_context, + kind=SpanKind.PRODUCER, + attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "publish"}, + ): + carrier = {} + propagate.inject(carrier) + payload.update(carrier) + channel.basic_publish( + exchange="", + routing_key=QUEUE_NAME, + body=encode_job(payload), + properties=pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(row["id"]), + type="attachment.process.v1", + ), + mandatory=True, + ) + conn.execute( + "UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + return True + + +def main() -> None: + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-publisher")) + store = Store() + store.open() + try: + while True: + connection = None + try: + connection = connect() + channel = connection.channel() + declare_queues(channel) + channel.confirm_delivery() + store.recover_expired_attachment_jobs( + int(os.getenv("JOB_MAX_ATTEMPTS", "5")), + int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS, + ) + if not publish_pending(store, channel): + time.sleep(1) + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("outbox publishing cycle failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..421e046 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import logging +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +LOGGER = logging.getLogger("messenger.reconcile") + + +def reconcile( + store: Store, + grace_seconds: int, + apply_changes: bool, + restore_dir: Optional[Path] = None, +): + started = time.monotonic() + metadata = store.reconciliation_snapshot() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + found = set() + orphans = [] + for key, last_modified in store.objects.objects(): + found.add(key) + if key not in metadata and last_modified <= cutoff: + orphans.append(key) + + missing = [ + row for key, row in metadata.items() + if key not in found and row.get("reconcile_type", "original") == "original" + ] + restore_candidates = [] + if restore_dir is not None: + root = restore_dir.resolve(strict=True) + for row in missing: + source = restore_dir / str(row["id"]) + try: + resolved = source.resolve(strict=True) + if ( + resolved.parent != root + or not resolved.is_file() + or resolved.stat().st_size != row["size_bytes"] + ): + continue + content = resolved.read_bytes() + except OSError: + continue + if ( + len(content) == row["size_bytes"] + and hashlib.sha256(content).hexdigest() == row["sha256"].strip() + ): + restore_candidates.append((row, content)) + + restored = 0 + if apply_changes: + for key in orphans: + store.objects.delete(key) + restorable_ids = set() + for row, content in restore_candidates: + store.objects.put(row["object_key"], content, row["content_type"]) + store.clear_missing_object(row["id"]) + restorable_ids.add(row["id"]) + restored += 1 + for row in missing: + if row["id"] not in restorable_ids: + store.record_missing_object(row["id"]) + for key, row in metadata.items(): + if key in found and row.get("reconcile_type", "original") == "original": + store.clear_missing_object(row["id"]) + + unresolved_missing = len(missing) - restored + return { + "mode": "apply" if apply_changes else "dry-run", + "grace_seconds": grace_seconds, + "objects_seen": len(found), + "metadata_seen": len({str(row["id"]) for row in metadata.values()}), + "orphan_objects": len([key for key in found if key not in metadata]), + "stale_orphan_candidates": len(orphans), + "deleted_orphans": len(orphans) if apply_changes else 0, + "missing_objects": len(missing), + "unresolved_missing_objects": unresolved_missing, + "restorable_missing_objects": len(restore_candidates), + "restored_missing_objects": restored, + "missing_incidents_recorded": len(missing) - restored if apply_changes else 0, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + } + + +def main(): + parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects") + parser.add_argument( + "--restore-dir", + type=Path, + help="optional trusted backup directory with exact original bytes named by attachment UUID", + ) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error("--grace-seconds must be at least 60") + + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + store = Store() + store.open() + try: + result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir) + print(json.dumps(result, sort_keys=True)) + if result["unresolved_missing_objects"]: + LOGGER.error( + "%s database records reference missing S3 objects; metadata was retained for manual recovery", + result["unresolved_missing_objects"], + ) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/replay_attachment.py b/src/replay_attachment.py new file mode 100644 index 0000000..62ba8bf --- /dev/null +++ b/src/replay_attachment.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import argparse +import json +import uuid + +from .store import Store + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Explicitly replay a failed attachment or start a newer pipeline version." + ) + parser.add_argument("--attachment-id", required=True, type=uuid.UUID) + parser.add_argument( + "--pipeline-version", + required=True, + type=int, + help="pipeline version implemented by the deployed worker", + ) + args = parser.parse_args() + if args.pipeline_version < 1: + parser.error("--pipeline-version must be positive") + + store = Store() + store.open() + try: + result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version) + print(json.dumps(result, sort_keys=True)) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..6ec821f --- /dev/null +++ b/src/security.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import secrets +import uuid +from datetime import datetime, timezone + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + salt_text = base64.urlsafe_b64encode(salt).decode("ascii") + hash_text = base64.urlsafe_b64encode(derived).decode("ascii") + return f"scrypt$32768$8$1${salt_text}${hash_text}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError, UnicodeError): + return False diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..60fd3d5 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..ecc2764 --- /dev/null +++ b/src/store.py @@ -0,0 +1,848 @@ +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import secrets +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, BinaryIO, Dict, List, Optional, Tuple + +import redis +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + IDEMPOTENCY_TTL_SECONDS, + SESSION_TTL_SECONDS, +) +from .errors import ( + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .object_store import ObjectStore +from .pipeline_versions import PIPELINE_QUALITY +from .security import isoformat, utc_now, valid_uuid, verify_password + +LOGGER = logging.getLogger("messenger.store") + + +def database_url() -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + file_path = os.getenv("DATABASE_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE") + + +class Store: + """PostgreSQL metadata, S3 objects, and optional Redis session cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + self.objects = ObjectStore() + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + self.objects.ready() + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + "attachment_ids": attachment_ids or [], + } + + @staticmethod + def attachment(row: Dict[str, Any]) -> Dict[str, Any]: + variants = row.get("variants") or [] + public_variants = [ + { + name: variant[name] + for name in ("name", "content_type", "size_bytes", "width", "height", "sha256") + } + for variant in variants + ] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"].strip(), + "status": row.get("status", "queued"), + "variants": public_variants, + "error_code": row.get("error_code"), + "created_at": isoformat(row["created_at"]), + "updated_at": isoformat(row["updated_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + if not row or not verify_password(password, row["password_hash"]): + return None + return self.user(row) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_idempotent_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: List[str], + idempotency_key: str, + canonical_body: str, + ) -> Optional[Dict[str, Any]]: + request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() + lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") + lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + if attachment_ids: + attachments = conn.execute( + "SELECT id, status FROM attachments " + "WHERE chat_id = %s AND id = ANY(%s) FOR SHARE", + (chat_id, attachment_ids), + ).fetchall() + if len(attachments) != len(attachment_ids): + return None + if any(item["status"] == "failed" for item in attachments): + raise AttachmentNotReadyError + + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = conn.execute( + "SELECT request_hash, response, expires_at " + "FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " + "FOR UPDATE", + (sender_id, chat_id, idempotency_key), + ).fetchone() + if existing and existing["expires_at"] > utc_now(): + if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): + raise IdempotencyConflictError + return existing["response"] + if existing: + conn.execute( + "DELETE FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", + (sender_id, chat_id, idempotency_key), + ) + + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, ordinal) " + "VALUES (%s, %s, %s)", + (row["id"], attachment_id, ordinal), + ) + message = self.message(row, attachment_ids) + conn.execute( + "INSERT INTO idempotency_records " + "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " + "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False, separators=(",", ":")), + utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), + ), + ) + return message + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + ids_by_message = {} + if rows: + links = conn.execute( + "SELECT message_id, attachment_id FROM message_attachments " + "WHERE message_id = ANY(%s) ORDER BY message_id, ordinal", + ([row["id"] for row in rows],), + ).fetchall() + for link in links: + ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"])) + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor + + def can_access_chat(self, chat_id: str, user_id: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() is not None + + def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + content: BinaryIO, + size_bytes: int, + digest: str, + traceparent: Optional[str] = None, + tracestate: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + if not self.can_access_chat(chat_id, owner_id): + return None + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"original/{attachment_id}" + content.seek(0) + self.objects.put(staging_key, content, content_type, size_bytes) + try: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO attachments " + "(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, staging_key, job_id) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " + "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "status, variants, error_code, created_at, updated_at", + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + digest, + object_key, + staging_key, + job_id, + ), + ).fetchone() + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " + "VALUES (%s, %s, %s, 1, 1, %s, %s)", + (uuid.uuid4(), job_id, attachment_id, traceparent, tracestate), + ) + return self.attachment(row) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "attachment staging compensation failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + + def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + return self.attachment(row) if row else None + + def download_attachment( + self, attachment_id: str, user_id: str, variant: str = "original" + ) -> Optional[Tuple[str, bytes]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + if not row: + return None + if row["status"] != "ready": + raise AttachmentNotReadyError + if variant == "original": + key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"] + else: + match = next( + (item for item in (row["variants"] or []) if item["name"] == variant), + None, + ) + if match is None: + return None + key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"] + return content_type, self.objects.get(key, size_bytes) + + def claim_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int + ) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, " + "object_key, content_type, size_bytes, sha256 " + "FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + not row + or str(row["job_id"]) != job_id + or row["pipeline_version"] != pipeline_version + or row["status"] in ("ready", "failed") + or row["attempt"] != attempt + ): + return None + if row["status"] == "processing" and row["lease_until"] is not None: + if row["lease_until"] > utc_now(): + return None + return conn.execute( + "UPDATE attachments SET status = 'processing', " + "lease_until = clock_timestamp() + (%s * interval '1 second'), " + "updated_at = clock_timestamp() " + "WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, " + "size_bytes, sha256, attempt, pipeline_version", + (lease_seconds, attachment_id), + ).fetchone() + + def complete_attachment_job( + self, + attachment_id: str, + job_id: str, + pipeline_version: int, + attempt: int, + variants: List[Dict[str, Any]], + ) -> bool: + with self.pool.connection() as conn: + with conn.transaction(): + updated = conn.execute( + "UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, " + "lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s " + "AND status = 'processing' AND lease_until > clock_timestamp()", + ( + json.dumps(variants, separators=(",", ":")), + attachment_id, + job_id, + pipeline_version, + attempt, + ), + ) + return updated.rowcount == 1 + + def fail_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, + max_attempts: int, error_code: str, retry_delay_seconds: Optional[float], + traceparent: Optional[str] = None, tracestate: Optional[str] = None, + ) -> None: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT status, attempt FROM attachments " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE", + (attachment_id, job_id, pipeline_version), + ).fetchone() + if not row or row["status"] != "processing" or row["attempt"] != attempt: + return + if retry_delay_seconds is not None and attempt < max_attempts: + next_attempt = attempt + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, error_code, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) " + "VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) " + "ON CONFLICT (job_id, attempt) DO NOTHING", + ( + uuid.uuid4(), job_id, attachment_id, pipeline_version, + next_attempt, retry_delay_seconds, traceparent, tracestate, + ), + ) + else: + final_code = "retry_exhausted" if attempt >= max_attempts else error_code + conn.execute( + "UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, " + "updated_at = clock_timestamp() WHERE id = %s", + (final_code, attachment_id), + ) + + def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]: + if pipeline_version not in PIPELINE_QUALITY: + raise ValueError("pipeline version is not implemented by this deployment") + with self.pool.connection() as conn: + current = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, " + "content_type, size_bytes, sha256 FROM attachments WHERE id = %s", + (attachment_id,), + ).fetchone() + if current is None: + raise ValueError("attachment not found") + if pipeline_version < current["pipeline_version"]: + raise ValueError("pipeline version cannot move backwards") + if pipeline_version == current["pipeline_version"] and current["status"] != "failed": + raise ValueError("same-version replay is allowed only for failed attachments") + if current["status"] not in ("queued", "processing", "ready", "failed"): + raise ValueError("attachment cannot be replayed in its current state") + + try: + content = self.objects.get(current["object_key"], current["size_bytes"]) + except ObjectMissingError as object_error: + try: + content = self.objects.get(current["staging_key"], current["size_bytes"]) + except ObjectMissingError: + raise object_error + if ( + len(content) != current["size_bytes"] + or hashlib.sha256(content).hexdigest() != current["sha256"].strip() + ): + raise ValueError("stored original does not match attachment metadata") + + prior_job_id = str(current["job_id"]) + next_job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}/{next_job_id}" + self.objects.put(staging_key, content, current["content_type"], len(content)) + try: + with self.pool.connection() as conn: + with conn.transaction(): + locked = conn.execute( + "SELECT job_id, pipeline_version, attempt, status FROM attachments " + "WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + locked is None + or str(locked["job_id"]) != prior_job_id + or locked["pipeline_version"] != current["pipeline_version"] + or locked["attempt"] != current["attempt"] + or locked["status"] != current["status"] + ): + raise RuntimeError("attachment changed while preparing replay") + conn.execute( + "UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, " + "status = 'queued', staging_key = %s, variants = '[]'::jsonb, " + "error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s", + (next_job_id, pipeline_version, staging_key, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, 1)", + (uuid.uuid4(), next_job_id, attachment_id, pipeline_version), + ) + conn.execute( + "INSERT INTO attachment_pipeline_events " + "(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) " + "VALUES (%s, %s, %s, %s, %s, 'replay')", + ( + attachment_id, + prior_job_id, + next_job_id, + current["pipeline_version"], + pipeline_version, + ), + ) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "replay staging cleanup failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + return { + "attachment_id": str(attachment_id), + "previous_job_id": prior_job_id, + "job_id": str(next_job_id), + "pipeline_version": pipeline_version, + "attempt": 1, + "status": "queued", + } + + def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + "SELECT id, job_id, pipeline_version, attempt FROM attachments " + "WHERE status = 'processing' AND lease_until < clock_timestamp() " + "ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100" + ).fetchall() + for row in rows: + if row["attempt"] >= max_attempts: + conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', " + "lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + continue + next_attempt = row["attempt"] + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, row["id"]), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING", + (uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt), + ) + return len(rows) + + def claim_outbox(self, conn): + return conn.execute( + "SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, " + "o.traceparent, o.tracestate, " + "a.created_at FROM attachment_outbox o " + "JOIN attachments a ON a.id = o.attachment_id " + "WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() " + "ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1" + ).fetchone() + + def queue_snapshot(self) -> Tuple[int, float]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT count(*) AS depth, " + "COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age " + "FROM attachments WHERE status = 'queued'" + ).fetchone() + return int(row["depth"]), float(row["oldest_age"]) + + def reconciliation_snapshot(self): + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, " + "size_bytes, sha256, content_type FROM attachments" + ).fetchall() + expected = {} + for row in rows: + if row["status"] == "ready": + expected[row["object_key"]] = {**row, "reconcile_type": "original"} + for variant in row["variants"] or []: + expected[variant["object_key"]] = { + **row, + "object_key": variant["object_key"], + "size_bytes": variant["size_bytes"], + "sha256": variant["sha256"], + "content_type": variant["content_type"], + "reconcile_type": "variant", + } + elif row["status"] in ("queued", "processing"): + expected[row["staging_key"]] = {**row, "reconcile_type": "staging"} + expected[row["object_key"]] = {**row, "reconcile_type": "inflight"} + for name, extension in ( + ("thumbnail", "jpg"), + ("crop", "jpg"), + ("optimized", "webp"), + ): + key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}" + expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"} + return expected + + def record_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO reconciliation_incidents (attachment_id, incident_type) " + "VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) " + "DO UPDATE SET last_seen_at = clock_timestamp()", + (attachment_id,), + ) + + def clear_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM reconciliation_incidents " + "WHERE attachment_id = %s AND incident_type = 'missing_object'", + (attachment_id,), + ) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..9fb6572 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,296 @@ +import contextvars +import json +import logging +import os +import platform +import socket +import threading +import time +from datetime import datetime, timezone +from typing import Optional + +import psycopg +from opentelemetry import metrics, trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON, ParentBased, TraceIdRatioBased +from opentelemetry.instrumentation.utils import suppress_instrumentation + +REQUEST_ID = contextvars.ContextVar("request_id", default="") +_lock = threading.Lock() +_runtime = { + "pool_used": 0, + "pool_waiting": 0, + "queue_depth": 0, + "queue_oldest_seconds": 0.0, + "worker_busy": 0, + "worker_capacity": 0, + "cpu": 0.0, + "memory": 0, +} +_provider = None +_meter = None +_service_name = "messenger" +_enabled = True +_request_counter = None +_request_duration = None +_job_counter = None +_job_duration = None +_time_to_ready = None +_queue_depth = None +_queue_oldest = None +_pool_used = None +_pool_waiting = None +_worker_busy = None +_worker_capacity = None +_process_cpu = None +_process_memory = None + + +class JsonLogFormatter(logging.Formatter): + converter = time.gmtime + + def format(self, record): + span_context = trace.get_current_span().get_span_context() + document = { + "timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(), + "severity": record.levelname, + "service.name": _service_name, + "event": record.getMessage(), + "request_id": REQUEST_ID.get(), + } + if span_context.is_valid: + document["trace_id"] = format(span_context.trace_id, "032x") + document["span_id"] = format(span_context.span_id, "016x") + if record.exc_info: + document["exception.type"] = record.exc_info[0].__name__ + return json.dumps(document, ensure_ascii=True, separators=(",", ":")) + + +def _sampler(): + if os.getenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") == "always_off": + return ALWAYS_OFF + if os.getenv("OTEL_TRACES_SAMPLER", "") == "always_on": + return ALWAYS_ON + ratio = float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0")) + if not 0 <= ratio <= 1: + raise ValueError("OTEL_TRACES_SAMPLER_ARG must be between 0 and 1") + return ParentBased(TraceIdRatioBased(ratio)) + + +def initialize(service_name: str) -> None: + global _enabled + global _service_name + global _provider, _meter + global _request_counter, _request_duration, _job_counter, _job_duration, _time_to_ready + global _queue_depth, _queue_oldest, _pool_used, _pool_waiting + global _worker_busy, _worker_capacity, _process_cpu, _process_memory + + enabled = os.getenv("OTEL_SDK_DISABLED", "false").lower() != "true" + _enabled = enabled + _service_name = service_name + instance_id = os.getenv("INSTANCE_ID", socket.gethostname()) + resource = Resource.create( + { + "service.name": service_name, + "service.version": os.getenv("SERVICE_VERSION", "lab6"), + "service.instance.id": instance_id, + "deployment.environment.name": os.getenv("APP_ENV", "lab"), + "host.name": socket.gethostname(), + "os.type": platform.system().lower(), + } + ) + if enabled: + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") + _provider = TracerProvider(resource=resource, sampler=_sampler()) + _provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint.rstrip("/") + "/v1/traces"), + max_queue_size=4096, + max_export_batch_size=512, + schedule_delay_millis=1000, + export_timeout_millis=10000, + ) + ) + trace.set_tracer_provider(_provider) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint.rstrip("/") + "/v1/metrics"), + export_interval_millis=10000, + export_timeout_millis=5000, + ) + metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[reader])) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint.rstrip("/") + "/v1/logs"), + max_queue_size=4096, + max_export_batch_size=512, + schedule_delay_millis=1000, + export_timeout_millis=10000, + ) + ) + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + else: + trace.set_tracer_provider(TracerProvider(resource=resource, sampler=ALWAYS_OFF)) + metrics.set_meter_provider(MeterProvider(resource=resource)) + otel_handler = None + + root = logging.getLogger() + root.setLevel(os.getenv("LOG_LEVEL", "INFO")) + formatter = JsonLogFormatter() + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + root.handlers[:] = [stream_handler] + if otel_handler is not None: + root.addHandler(otel_handler) + logging.getLogger("pika").setLevel(logging.WARNING) + + if enabled: + try: + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + + BotocoreInstrumentor().instrument() + PsycopgInstrumentor().instrument() + except ImportError as exc: + raise RuntimeError("OTel database and S3 instrumentation dependencies are required") from exc + + _meter = metrics.get_meter("messenger.telemetry") + _request_counter = _meter.create_counter("messenger.http.requests", unit="{request}") + _request_duration = _meter.create_histogram("messenger.http.duration", unit="s") + _job_counter = _meter.create_counter("messenger.jobs.processed", unit="{job}") + _job_duration = _meter.create_histogram("messenger.jobs.duration", unit="s") + _time_to_ready = _meter.create_histogram("messenger.attachments.time_to_ready", unit="s") + + def observe(name, attributes=None): + def callback(_options): + with _lock: + return [metrics.Observation(_runtime[name], attributes=attributes or {})] + + return callback + + _queue_depth = _meter.create_observable_gauge( + "messenger.queue.depth", + callbacks=[observe("queue_depth", {"queue.name": "attachments.process.v1"})], + unit="{job}", + ) + _queue_oldest = _meter.create_observable_gauge( + "messenger.queue.oldest_age", + callbacks=[observe("queue_oldest_seconds", {"queue.name": "attachments.process.v1"})], + unit="s", + ) + _pool_used = _meter.create_observable_gauge( + "messenger.db.pool.connections.used", + callbacks=[observe("pool_used", {"pool.name": "primary"})], + unit="{connection}", + ) + _pool_waiting = _meter.create_observable_gauge( + "messenger.db.pool.requests.waiting", + callbacks=[observe("pool_waiting", {"pool.name": "primary"})], + unit="{request}", + ) + _worker_busy = _meter.create_observable_gauge( + "messenger.worker.busy", callbacks=[observe("worker_busy")], unit="{worker}" + ) + _worker_capacity = _meter.create_observable_gauge( + "messenger.worker.capacity", callbacks=[observe("worker_capacity")], unit="{worker}" + ) + _process_cpu = _meter.create_observable_gauge( + "messenger.process.cpu.utilization", callbacks=[observe("cpu")], unit="1" + ) + _process_memory = _meter.create_observable_gauge( + "messenger.process.memory.usage", callbacks=[observe("memory")], unit="By" + ) + + +def tracer(): + return trace.get_tracer("messenger.application") + + +def record_http_request(method: str, route: str, status: int, duration: float) -> None: + attributes = { + "http.request.method": method, + "http.route": route, + "status_class": f"{status // 100}xx", + } + if _request_counter is not None: + _request_counter.add(1, attributes) + _request_duration.record(duration, attributes) + if _enabled and not route.startswith("/health/"): + logging.getLogger("messenger.http").info( + "http request completed method=%s route=%s status_class=%s", + method, + route, + f"{status // 100}xx", + ) + + +def record_job(outcome: str, duration: float, time_to_ready: Optional[float] = None) -> None: + attributes = {"job.type": "attachment.process.v1", "outcome": outcome} + if _job_counter is not None: + _job_counter.add(1, attributes) + _job_duration.record(duration, attributes) + if time_to_ready is not None: + _time_to_ready.record(time_to_ready, {"outcome": outcome}) + + +def update_runtime(**values) -> None: + with _lock: + _runtime.update(values) + + +def start_runtime_metrics(store, worker_capacity: int = 0) -> Optional[threading.Thread]: + if not _enabled: + return None + + def refresh(): + try: + import psutil + + process = psutil.Process() + process.cpu_percent(interval=None) + while True: + try: + stats = store.pool.get_stats() + with suppress_instrumentation(): + queue_depth, oldest_age = store.queue_snapshot() + cpu = process.cpu_percent(interval=1) / 100.0 + memory = process.memory_info().rss + update_runtime( + pool_used=max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)), + pool_waiting=stats.get("requests_waiting", 0), + queue_depth=queue_depth, + queue_oldest_seconds=oldest_age, + worker_capacity=worker_capacity, + cpu=cpu, + memory=memory, + ) + except (psycopg.Error, OSError, RuntimeError, ValueError): + logging.getLogger("messenger.telemetry").exception("runtime metric sample failed") + time.sleep(9) + except (OSError, RuntimeError, ValueError): + logging.getLogger("messenger.telemetry").exception("runtime metric sampler stopped") + raise + + thread = threading.Thread(target=refresh, name="runtime-metrics", daemon=True) + thread.start() + return thread + + +def worker_started() -> None: + with _lock: + _runtime["worker_busy"] += 1 + + +def worker_stopped() -> None: + with _lock: + _runtime["worker_busy"] = max(0, _runtime["worker_busy"] - 1) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..0828d6c --- /dev/null +++ b/src/worker.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +import hashlib +import io +import json +import logging +import multiprocessing +import os +import random +import time +import uuid +import warnings +import zlib +from datetime import datetime, timezone +from typing import Callable, Optional + +import pika +import psycopg +from opentelemetry import propagate +from opentelemetry.trace import SpanKind, Status, StatusCode +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, ImageOps, UnidentifiedImageError + +from .config import MAX_IMAGE_PIXELS +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, declare_queues +from .pipeline_versions import PIPELINE_QUALITY +from .store import Store +from .telemetry import ( + initialize, + record_job, + start_runtime_metrics, + tracer, + worker_started, + worker_stopped, +) + +LOGGER = logging.getLogger("messenger.worker") + +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1")) + + +class InvalidAttachment(Exception): + pass + + +class UnsupportedPipeline(InvalidAttachment): + pass + + +def _rgb_image(content: bytes) -> Image.Image: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(content)) as source: + if source.format not in ("PNG", "JPEG"): + raise InvalidAttachment("unsupported_image") + width, height = source.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise InvalidAttachment("image_too_large") + source.load() + image = ImageOps.exif_transpose(source) + try: + if "A" in image.getbands() or "transparency" in image.info: + with image.convert("RGBA") as rgba: + background = Image.new("RGB", rgba.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + finally: + if image is not source: + image.close() + except InvalidAttachment: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError, + UnidentifiedImageError, ValueError, zlib.error) as exc: + raise InvalidAttachment("invalid_image") from exc + + +def _encode(image: Image.Image, image_format: str, **options) -> bytes: + output = io.BytesIO() + image.save(output, format=image_format, **options) + return output.getvalue() + + +def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1): + quality = PIPELINE_QUALITY.get(pipeline_version) + if quality is None: + raise UnsupportedPipeline("unsupported_pipeline_version") + image = _rgb_image(content) + try: + variants = [] + thumbnail = image.copy() + try: + thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes)) + finally: + thumbnail.close() + + crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + try: + crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("crop", "image/jpeg", 128, 128, crop_bytes)) + finally: + crop.close() + + optimized = image.copy() + try: + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6) + variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes)) + finally: + optimized.close() + + result = [] + for name, content_type, width, height, data in variants: + extension = "webp" if content_type == "image/webp" else "jpg" + key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}" + result.append( + { + "name": name, + "content_type": content_type, + "size_bytes": len(data), + "width": width, + "height": height, + "sha256": hashlib.sha256(data).hexdigest(), + "object_key": key, + "content": data, + } + ) + return result + finally: + image.close() + + +class ProcessingTimeout(TimeoutError): + pass + + +def _variant_process(connection, attachment_id, content, pipeline_version, builder): + try: + connection.send(("ok", builder(attachment_id, content, pipeline_version))) + except InvalidAttachment as exc: + connection.send(("invalid", str(exc))) + except Exception as exc: + connection.send(("error", type(exc).__name__)) + finally: + connection.close() + + +def _build_variants_bounded( + attachment_id: str, + content: bytes, + pipeline_version: int, + timeout_seconds: int, + builder=build_variants, +): + if timeout_seconds < 1: + raise ValueError("timeout_seconds must be positive") + context = multiprocessing.get_context("spawn") + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_variant_process, + args=(sender, attachment_id, content, pipeline_version, builder), + ) + process.start() + sender.close() + try: + if not receiver.poll(timeout_seconds): + process.terminate() + process.join(timeout=2) + if process.is_alive(): + process.kill() + process.join() + raise ProcessingTimeout("image processing exceeded its time budget") + try: + result, value = receiver.recv() + except EOFError as exc: + raise RuntimeError("image processing process exited without a result") from exc + process.join(timeout=2) + if process.is_alive(): + process.terminate() + process.join() + if result == "invalid": + raise InvalidAttachment(value) + if result == "error": + raise RuntimeError(f"image processing failed in subprocess ({value})") + return value + finally: + receiver.close() + if process.is_alive(): + process.terminate() + process.join() + + +def _job(payload: bytes): + value = json.loads(payload) + required = { + "schema_version", "job_id", "type", "attachment_id", + "pipeline_version", "attempt", "created_at", + } + if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}): + raise ValueError("invalid job fields") + if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1": + raise ValueError("invalid job schema") + uuid.UUID(value["job_id"]) + uuid.UUID(value["attachment_id"]) + if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1: + raise ValueError("invalid pipeline version") + if not isinstance(value["attempt"], int) or value["attempt"] < 1: + raise ValueError("invalid job attempt") + return value + + +def _process( + store: Store, + job: dict, + fault_injector: Optional[Callable[[str], None]] = None, +) -> str: + def inject(point: str) -> None: + if fault_injector is not None: + fault_injector(point) + + attachment_id = job["attachment_id"] + row = store.claim_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS, + ) + if row is None: + LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"]) + return "skipped" + + try: + inject("after_claim") + with tracer().start_as_current_span("attachment.read_staging"): + content = store.objects.get(row["staging_key"], row["size_bytes"]) + if hashlib.sha256(content).hexdigest() != row["sha256"].strip(): + raise InvalidAttachment("checksum_mismatch") + variants = _build_variants_bounded( + attachment_id, + content, + job["pipeline_version"], + JOB_TIMEOUT_SECONDS, + ) + inject("before_original_put") + with tracer().start_as_current_span("attachment.write_original"): + store.objects.put(row["object_key"], content, row["content_type"], len(content)) + inject("after_original_put") + persisted = [] + for variant in variants: + data = variant.pop("content") + inject(f"before_variant_put:{variant['name']}") + with tracer().start_as_current_span("attachment.write_variant"): + store.objects.put(variant["object_key"], data, variant["content_type"], len(data)) + inject(f"after_variant_put:{variant['name']}") + persisted.append(variant) + inject("before_commit") + with tracer().start_as_current_span("attachment.commit_ready"): + completed = store.complete_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + persisted, + ) + if not completed: + LOGGER.warning("job lease was superseded before completion: %s", job["job_id"]) + return "skipped" + inject("after_commit") + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id) + LOGGER.info("attachment processing completed: %s", attachment_id) + return "success" + except InvalidAttachment as exc: + carrier = {} + propagate.inject(carrier) + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, str(exc), None, carrier.get("traceparent"), carrier.get("tracestate"), + ) + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id) + LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc) + return "failed" + except Exception as exc: + delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25) + carrier = {} + propagate.inject(carrier) + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, "processing_failed", delay, carrier.get("traceparent"), carrier.get("tracestate"), + ) + LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__) + return "failed" + + +def handle_delivery( + store: Store, + channel: pika.adapters.blocking_connection.BlockingChannel, + method, + body: bytes, + fault_injector: Optional[Callable[[str], None]] = None, +) -> None: + try: + job = _job(body) + except (ValueError, TypeError, json.JSONDecodeError): + LOGGER.exception("invalid broker job rejected") + channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + return + parent_context = propagate.extract( + {key: job[key] for key in ("traceparent", "tracestate") if key in job} + ) + started = time.perf_counter() + created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")) + with tracer().start_as_current_span( + "attachment.process", + context=parent_context, + kind=SpanKind.CONSUMER, + attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "process"}, + ) as span: + worker_started() + try: + outcome = _process(store, job, fault_injector) + if outcome == "failed": + span.set_status(Status(StatusCode.ERROR)) + if outcome != "skipped": + elapsed = time.perf_counter() - started + ready_duration = ( + max(0.0, datetime.now(timezone.utc).timestamp() - created_at.timestamp()) + if outcome == "success" + else None + ) + record_job(outcome, elapsed, ready_duration) + finally: + worker_stopped() + if fault_injector is not None: + fault_injector("before_ack") + channel.basic_ack(delivery_tag=method.delivery_tag) + + +def main() -> None: + if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1: + raise RuntimeError("Worker limits must be positive") + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-worker")) + store = Store() + store.open() + start_runtime_metrics(store, WORKER_CONCURRENCY) + connection = None + try: + while True: + try: + connection = connect() + channel = connection.channel() + declare_queues(channel) + channel.basic_qos(prefetch_count=WORKER_CONCURRENCY) + + def on_message(ch, method, properties, body): + handle_delivery(store, ch, method, body) + + channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False) + LOGGER.info("worker consuming durable attachment queue") + channel.start_consuming() + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("worker connection failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + connection = None + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..1663aaa --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Negative attachment and cross-chat authorization checks against a running lab4 API.""" +import uuid +from pathlib import Path +import sys +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202): + return client.call( + "POST", + "/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""), + expected, + actor=actor, + raw=data, + headers={"Content-Type": content_type}, + schema="Attachment" if expected in (201, 202) else None, + ) + + +def main(): + args = arguments("Attachment upload, size, MIME, and chat authorization checks") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + other_chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + print("Checking accepted fixture upload.", flush=True) + attachment = upload(client, alice, chat["id"], image)["json"] + + print("Checking damaged and spoofed image rejection.", flush=True) + upload(client, alice, chat["id"], b"not an image", expected=415) + upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415) + print("Checking byte and pixel limits.", flush=True) + upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413) + import struct + import zlib + + def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30")) + bad_idat[-3] ^= 0xFF + damaged_idat = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", bytes(bad_idat)) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], damaged_idat, expected=415) + + too_many_pixels = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], too_many_pixels, expected=413) + print("Checking filename and cross-chat authorization.", flush=True) + upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400) + upload(client, eve, chat["id"], image, expected=404) + client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve) + client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + 404, + actor=alice, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": "", "attachment_ids": [attachment["id"]]}, + ) + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=bob, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": ""}, + ) + print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_idempotency_race.py b/student-tests/test_idempotency_race.py new file mode 100644 index 0000000..646869a --- /dev/null +++ b/student-tests/test_idempotency_race.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a concurrent idempotency-key race through the load balancer.""" +from concurrent.futures import ThreadPoolExecutor +import threading +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency integration check") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = "race-" + uuid.uuid4().hex + text = "concurrent-" + uuid.uuid4().hex + count = 12 + barrier = threading.Barrier(count) + + def send_once(_): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=10) + return worker.message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send_once, range(count))) + + instance_ids = {response["headers"]["X-Instance-Id"] for response in responses} + messages = [response["json"] for response in responses] + check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas") + check(all(message == messages[0] for message in messages), "same-key requests returned different messages") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"] + matches = [message for message in page["items"] if message["text"] == text] + check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages") + print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_outbox_recovery_integration.py b/student-tests/test_outbox_recovery_integration.py new file mode 100644 index 0000000..972228b --- /dev/null +++ b/student-tests/test_outbox_recovery_integration.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify an upload committed while the publisher is stopped is processed after restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify committed attachment outbox intent survives publisher downtime") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + compose("stop", "publisher") + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted into queued state") + finally: + compose("start", "publisher") + + ready = client.wait_ready(alice, attachment) + stored_original = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + thumbnail = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + variant = next(item for item in ready["variants"] if item["name"] == "thumbnail") + check(stored_original == original, "recovered job changed original bytes") + check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch") + print( + json.dumps( + { + "result": "PASS", + "accepted_status_before_publisher_restart": attachment["status"], + "final_status": ready["status"], + "attachment_id": attachment["id"], + "original_sha256": hashlib.sha256(stored_original).hexdigest(), + "thumbnail_sha256": variant["sha256"], + "recovered_without_manual_republish": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..0d06e56 --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing.""" +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import uuid +from datetime import datetime, timezone + +os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret") + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import pika + +from src.publisher import publish_pending +from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery + + +ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001" +JOB_ID = "00000000-0000-4000-8000-000000000002" + + +class SimulatedCrash(BaseException): + pass + + +class FakeObjects: + def __init__(self, content): + self.data = {"staging/" + ATTACHMENT_ID: content} + self.puts = [] + + def get(self, key, max_bytes): + return self.data[key] + + def put(self, key, content, content_type, size_bytes=None): + value = content if isinstance(content, bytes) else content.read() + assert len(value) == size_bytes + self.data[key] = value + self.puts.append((key, hashlib.sha256(value).hexdigest())) + + def delete(self, key): + self.data.pop(key, None) + + +class FakeStore: + def __init__(self, content): + self.objects = FakeObjects(content) + self.row = { + "id": ATTACHMENT_ID, + "job_id": JOB_ID, + "pipeline_version": 1, + "attempt": 1, + "status": "queued", + "lease_until": 0.0, + "staging_key": "staging/" + ATTACHMENT_ID, + "object_key": "original/" + ATTACHMENT_ID, + "content_type": "image/png", + "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "variants": [], + } + + def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] in ("ready", "failed") + ): + return None + if row["status"] == "processing" and row["lease_until"] > time.monotonic(): + return None + row["status"] = "processing" + row["lease_until"] = time.monotonic() + lease_seconds + return dict(row) + + def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] != "processing" + or row["lease_until"] <= time.monotonic() + ): + return False + row["status"] = "ready" + row["lease_until"] = 0.0 + row["variants"] = variants + return True + + def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay): + row = self.row + if ( + row["id"] == attachment_id + and row["job_id"] == job_id + and row["pipeline_version"] == version + and row["attempt"] == attempt + and row["status"] == "processing" + ): + row["status"] = "failed" if delay is None else "queued" + row["lease_until"] = 0.0 + row["error_code"] = error_code + if delay is not None: + row["attempt"] += 1 + + def recover_expired(self): + if self.row["status"] == "processing": + self.row["attempt"] += 1 + self.row["status"] = "queued" + self.row["lease_until"] = 0.0 + + def replay(self, version): + row = self.row + previous_job = row["job_id"] + row["job_id"] = str(uuid.uuid4()) + row["pipeline_version"] = version + row["attempt"] = 1 + row["status"] = "queued" + row["lease_until"] = 0.0 + row["variants"] = [] + row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}" + self.objects.put( + row["staging_key"], + self.objects.data[row["object_key"]], + row["content_type"], + row["size_bytes"], + ) + return { + "schema_version": 1, + "job_id": row["job_id"], + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + "previous_job_id": previous_job, + } + + +class FakeChannel: + def __init__(self): + self.acked = [] + self.nacked = [] + + def basic_ack(self, delivery_tag): + self.acked.append(delivery_tag) + + def basic_nack(self, delivery_tag, requeue): + self.nacked.append((delivery_tag, requeue)) + + +class FakeMethod: + def __init__(self, delivery_tag): + self.delivery_tag = delivery_tag + + +def job_body(job_id=JOB_ID, version=1, attempt=1): + return json.dumps( + { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": attempt, + "created_at": "2026-01-01T00:00:00Z", + } + ).encode() + + +def crash_at(expected): + def inject(point): + if point == expected: + raise SimulatedCrash(point) + + return inject + + +def slow_builder(attachment_id, content, pipeline_version): + time.sleep(5) + + +def test_duplicate_and_ack_loss(content): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(1), job_body(), + fault_injector=crash_at("before_ack"), + ) + raise AssertionError("fault injection did not crash before ack") + except SimulatedCrash: + pass + assert store.row["status"] == "ready" + first_write_count = len(store.objects.puts) + assert channel.acked == [] + + handle_delivery(store, channel, FakeMethod(2), job_body()) + assert channel.acked == [2] + assert len(store.objects.puts) == first_write_count + assert store.row["status"] == "ready" + return { + "scenario": "duplicate_delivery_after_commit_before_ack", + "deliveries": 2, + "effective_object_writes": first_write_count, + "duplicate_delivery_writes": len(store.objects.puts) - first_write_count, + } + + +def test_crash_before_and_after_side_effect(content, crash_point): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(3), job_body(), + fault_injector=crash_at(crash_point), + ) + raise AssertionError(f"fault injection did not crash at {crash_point}") + except SimulatedCrash: + pass + assert channel.acked == [] + store.recover_expired() + retry = job_body(attempt=2) + handle_delivery(store, channel, FakeMethod(4), retry) + assert store.row["status"] == "ready" + assert store.row["attempt"] == 2 + assert channel.acked == [4] + assert store.objects.data["original/" + ATTACHMENT_ID] == content + return { + "scenario": "crash_" + crash_point, + "attempts": store.row["attempt"], + "terminal_status": store.row["status"], + } + + +def test_reordered_attempts(content): + store = FakeStore(content) + store.row["attempt"] = 2 + channel = FakeChannel() + handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2)) + writes_after_current = len(store.objects.puts) + handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1)) + assert store.row["status"] == "ready" + assert len(store.objects.puts) == writes_after_current + assert channel.acked == [5, 6] + return { + "scenario": "out_of_order_attempt_delivery", + "current_attempt": store.row["attempt"], + "stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current, + } + + +def test_stale_pipeline_cannot_publish_newer_result(content): + store = FakeStore(content) + channel = FakeChannel() + replayed = [] + + def supersede_after_first_variant(point): + if point == "after_variant_put:thumbnail" and not replayed: + replayed.append(store.replay(2)) + + handle_delivery( + store, channel, FakeMethod(7), job_body(), + fault_injector=supersede_after_first_variant, + ) + assert store.row["status"] == "queued" + assert store.row["pipeline_version"] == 2 + assert channel.acked == [7] + + current_job = replayed[0] + handle_delivery( + store, + channel, + FakeMethod(8), + job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]), + ) + assert store.row["status"] == "ready" + assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"} + assert channel.acked == [7, 8] + return { + "scenario": "stale_pipeline_after_newer_version", + "current_pipeline_version": store.row["pipeline_version"], + "published_variant_versions": sorted( + {variant["object_key"].split("/")[2] for variant in store.row["variants"]} + ), + } + + +class FakeTransaction: + def __init__(self, store): + self.store = store + + def __enter__(self): + self.published = False + self.store.current_transaction = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None and self.published: + self.store.outbox_published = True + self.store.current_transaction = None + return False + + +class FakeConnection: + def __init__(self, store): + self.store = store + + def transaction(self): + return FakeTransaction(self.store) + + def execute(self, query, params): + assert "UPDATE attachment_outbox" in query + assert params == ("outbox-row",) + self.store.current_transaction.published = True + + +class FakePool: + def __init__(self, store): + self.store = store + + def connection(self): + connection = FakeConnection(self.store) + + class ConnectionContext: + def __enter__(self_inner): + return connection + + def __exit__(self_inner, exc_type, exc_value, traceback): + return False + + return ConnectionContext() + + +class FakeOutboxStore: + def __init__(self): + self.pool = FakePool(self) + self.outbox_published = False + self.current_transaction = None + + def claim_outbox(self, connection): + if self.outbox_published: + return None + return { + "id": "outbox-row", + "job_id": JOB_ID, + "attachment_id": ATTACHMENT_ID, + "pipeline_version": 1, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + +class AmbiguousConfirmChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + self.messages.append(message) + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm") + + +class FailBeforePublishChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish") + self.messages.append(message) + + +def test_committed_outbox_survives_publisher_crash(): + store = FakeOutboxStore() + channel = FailBeforePublishChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher crash was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert channel.messages == [] + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 1 + return { + "scenario": "publisher_crash_before_publish", + "published_after_restart": len(channel.messages), + "outbox_recovered_after_restart": store.outbox_published, + } + + +def test_outbox_confirm_gap_republishes_same_job(): + store = FakeOutboxStore() + channel = AmbiguousConfirmChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher confirm fault was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 2 + first, second = channel.messages + assert first["body"] == second["body"] + assert first["properties"].message_id == second["properties"].message_id == "outbox-row" + return { + "scenario": "publish_confirm_lost_after_broker_accept", + "deliveries": len(channel.messages), + "same_message_id": True, + } + + +def test_image_processing_has_a_hard_timeout(): + started = time.monotonic() + try: + _build_variants_bounded( + ATTACHMENT_ID, + b"bounded timeout test", + 1, + timeout_seconds=1, + builder=slow_builder, + ) + raise AssertionError("slow transform exceeded its time budget without termination") + except ProcessingTimeout: + pass + elapsed = time.monotonic() - started + assert elapsed < 4 + return { + "scenario": "transform_timeout_terminated", + "configured_timeout_seconds": 1, + "observed_seconds": round(elapsed, 3), + } + + +def main(): + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + results = [ + test_duplicate_and_ack_loss(content), + test_crash_before_and_after_side_effect(content, "after_claim"), + test_crash_before_and_after_side_effect(content, "after_original_put"), + test_reordered_attempts(content), + test_stale_pipeline_cannot_publish_newer_result(content), + test_committed_outbox_survives_publisher_crash(), + test_outbox_confirm_gap_republishes_same_job(), + test_image_processing_has_a_hard_timeout(), + ] + report = { + "result": "PASS", + "track": "2", + "fixture": "tests/fixtures/sample.png", + "scenarios": results, + "invariants": { + "confirmed_work_eventually_ready": True, + "duplicate_effects_create_no_second_variant_set": True, + "stale_attempt_writes_no_effects": True, + "stale_pipeline_cannot_publish_current_metadata": True, + "ambiguous_confirm_republishes_stable_job": True, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + }, + } + evidence = ROOT / "evidence" / "pipeline-faults.json" + evidence.parent.mkdir(exist_ok=True) + evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, sort_keys=True)) + print(f"Evidence written to {evidence.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_pipeline_replay_integration.py b/student-tests/test_pipeline_replay_integration.py new file mode 100644 index 0000000..8687a95 --- /dev/null +++ b/student-tests/test_pipeline_replay_integration.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Verify explicit replay changes the deployed pipeline while preserving the original.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +MARK_FAILED_CODE = """ +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + updated = conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', " + "variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'", + (sys.argv[1],), + ) + if updated.rowcount != 1: + raise RuntimeError("test attachment was not ready for failed replay injection") +finally: + store.close() +""" + +AUDIT_CODE = """ +import json +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + rows = conn.execute( + "SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events " + "WHERE attachment_id = %s ORDER BY id", + (sys.argv[1],), + ).fetchall() + print(json.dumps(rows, sort_keys=True)) +finally: + store.close() +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def replay(container_id, attachment_id, version): + result = docker_exec( + container_id, + "python3", + "-m", + "src.replay_attachment", + "--attachment-id", + attachment_id, + "--pipeline-version", + str(version), + ) + return json.loads(result.stdout) + + +def main(): + args = arguments("Verify versioned attachment replay against the running Compose stack") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready_v1 = client.wait_ready(alice, attachment) + v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail") + v1_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch") + + containers = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.split() + check(bool(containers), "no running API container found") + replay_v2 = replay(containers[0], attachment["id"], 2) + check(replay_v2["status"] == "queued", "replay was not queued") + check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2") + check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id") + + ready_v2 = client.wait_ready(alice, ready_v1) + v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail") + v2_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding") + check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch") + check(original_bytes == original, "pipeline replay changed the original file") + + docker_exec( + containers[0], + "python3", + "-c", + MARK_FAILED_CODE, + attachment["id"], + ) + failed_replay = replay(containers[0], attachment["id"], 2) + check(failed_replay["status"] == "queued", "failed attachment replay was not queued") + check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version") + check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id") + ready_replayed = client.wait_ready(alice, ready_v2) + audit = json.loads( + docker_exec( + containers[0], + "python3", + "-c", + AUDIT_CODE, + attachment["id"], + ).stdout + ) + check( + [(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit] + == [(1, 2, "replay"), (2, 2, "replay")], + "pipeline replay audit trail is incomplete", + ) + check(ready_replayed["status"] == "ready", "failed attachment did not recover") + + print( + json.dumps( + { + "result": "PASS", + "previous_job_id": replay_v2["previous_job_id"], + "job_id": replay_v2["job_id"], + "pipeline_version": replay_v2["pipeline_version"], + "failed_replay_job_id": failed_replay["job_id"], + "audited_replays": len(audit), + "v1_thumbnail_sha256": v1_thumbnail["sha256"], + "v2_thumbnail_sha256": v2_thumbnail["sha256"], + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 0000000..ff12312 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Deterministic tests of the reconciliation decision rules without deleting real objects.""" +import os +import hashlib +import sys +import tempfile +from pathlib import Path + +os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret") +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from datetime import datetime, timedelta, timezone + +from src.reconcile import reconcile + + +class FakeObjects: + def __init__(self, items): + self.items = items + self.deleted = [] + + def objects(self): + return iter(self.items) + + def delete(self, key): + self.deleted.append(key) + self.items = [(name, date) for name, date in self.items if name != key] + + def put(self, key, content, content_type): + self.items.append((key, datetime.now(timezone.utc))) + + +class FakeStore: + def __init__(self): + now = datetime.now(timezone.utc) + self.metadata = { + "original/present": {"id": "present-id", "object_key": "original/present"}, + "original/missing": { + "id": "missing-id", + "object_key": "original/missing", + "size_bytes": 4, + "sha256": hashlib.sha256(b"test").hexdigest(), + "content_type": "image/png", + }, + } + self.objects = FakeObjects( + [ + ("original/present", now), + ("original/stale-orphan", now - timedelta(hours=2)), + ("original/new-orphan", now - timedelta(minutes=10)), + ] + ) + self.missing_incidents = set() + self.cleared_incidents = set() + + def reconciliation_snapshot(self): + return self.metadata + + def record_missing_object(self, attachment_id): + self.missing_incidents.add(attachment_id) + + def clear_missing_object(self, attachment_id): + self.missing_incidents.discard(attachment_id) + self.cleared_incidents.add(attachment_id) + + +def main(): + store = FakeStore() + dry_run = reconcile(store, grace_seconds=3600, apply_changes=False) + assert dry_run["stale_orphan_candidates"] == 1 + assert dry_run["missing_objects"] == 1 + assert dry_run["deleted_orphans"] == 0 + assert store.objects.deleted == [] and store.missing_incidents == set() + + applied = reconcile(store, grace_seconds=3600, apply_changes=True) + assert store.objects.deleted == ["original/stale-orphan"] + assert "original/new-orphan" not in store.objects.deleted + assert store.missing_incidents == {"missing-id"} + assert store.cleared_incidents == {"present-id"} + assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1 + + with tempfile.TemporaryDirectory() as backup: + restore_dir = Path(backup) + (restore_dir / "missing-id").write_bytes(b"test") + preview = reconcile(store, 3600, False, restore_dir) + assert preview["restorable_missing_objects"] == 1 + assert store.missing_incidents == {"missing-id"} + restored = reconcile(store, 3600, True, restore_dir) + assert restored["restored_missing_objects"] == 1 + assert store.missing_incidents == set() + + repeated = reconcile(store, grace_seconds=3600, apply_changes=True) + assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0 + assert store.objects.deleted == ["original/stale-orphan"] + assert store.missing_incidents == set() + print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_reconcile_integration.py b/student-tests/test_reconcile_integration.py new file mode 100644 index 0000000..78a719d --- /dev/null +++ b/student-tests/test_reconcile_integration.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects.""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +STORE_CODE = """ +import sys +from pathlib import Path +from src.store import Store + +store = Store() +store.open() +try: + action = sys.argv[1] + if action == "key": + attachment_id = sys.argv[2] + matches = [ + key for key, row in store.reconciliation_snapshot().items() + if str(row["id"]) == attachment_id and row["reconcile_type"] == "original" + ] + if len(matches) != 1: + raise RuntimeError("attachment metadata lookup failed") + print(matches[0]) + elif action == "put": + key, file_path = sys.argv[2:4] + content = Path(file_path).read_bytes() + store.objects.put(key, content, "image/png", len(content)) + elif action == "delete": + store.objects.delete(sys.argv[2]) + elif action == "exists": + print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no") + else: + raise RuntimeError("unsupported test operation") +finally: + store.close() +""" + +PRIVATE_CHECK_CODE = """ +import socket +import sys +import urllib.error +import urllib.request + +try: + socket.getaddrinfo("s3-core", 8888) +except socket.gaierror: + print("PASS: SeaweedFS master/filer network is isolated") +else: + raise RuntimeError("API network can resolve the SeaweedFS core service") + +for port in (8181, 9101): + try: + socket.create_connection(("s3", port), timeout=1).close() + except OSError: + continue + raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port)) + +try: + urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3) +except urllib.error.HTTPError as error: + if error.code == 403: + print("PASS: unauthenticated bucket access denied") + else: + raise RuntimeError("unexpected anonymous S3 status: " + str(error.code)) +else: + raise RuntimeError("private bucket allowed an unauthenticated GET") +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def reconciliation(container_id, *args): + result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args) + return json.loads(result.stdout) + + +def object_exists(container_id, key): + return docker_exec( + container_id, "python3", "-c", STORE_CODE, "exists", key + ).stdout.strip() == "yes" + + +def main(): + args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + containers = result.stdout.split() + check(len(containers) >= 2, "at least two running API replicas are required") + container_id = containers[0] + key = docker_exec( + container_id, "python3", "-c", STORE_CODE, "key", attachment["id"] + ).stdout.strip() + orphan_key = "original/" + str(uuid.uuid4()) + + with tempfile.TemporaryDirectory() as backup: + source = Path(backup) / "original.png" + source.write_bytes(content) + container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png" + recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex + recovery_file = recovery_dir + "/" + attachment["id"] + orphan_created = False + orphan_removed = False + missing_deleted = False + missing_restored = False + try: + subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True) + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file) + orphan_created = True + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key) + missing_deleted = True + + preview = reconciliation(container_id) + check( + preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1, + "dry-run missed injected inconsistencies", + ) + check(preview["deleted_orphans"] == 0, "dry-run modified S3") + immediate_apply = reconciliation(container_id, "--apply") + check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period") + + docker_exec(container_id, "mkdir", "-p", recovery_dir) + subprocess.run( + ["docker", "cp", str(source), container_id + ":" + recovery_file], + check=True, + ) + restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir) + check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable") + restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored") + missing_restored = True + + repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object") + check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period") + print("Waiting for the 60-second orphan grace period.") + time.sleep(61) + + cleanup_preview = reconciliation(container_id) + check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace") + cleaned = reconciliation(container_id, "--apply") + orphan_removed = not object_exists(container_id, orphan_key) + check(orphan_removed, "injected stale orphan was not removed") + final_scan = reconciliation(container_id, "--apply") + check( + final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0, + "repeat scan is not clean", + ) + finally: + if missing_deleted and not missing_restored: + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file) + if orphan_created and not orphan_removed: + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key) + + private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip() + check("PASS:" in private, "unauthenticated S3 access check failed") + downloaded = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(downloaded == content, "reconciler changed the original of an attachment used by a message") + check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference") + print( + json.dumps( + { + "result": "PASS", + "dataset_objects_scanned": final_scan["objects_seen"], + "metadata_records_scanned": final_scan["metadata_seen"], + "scan_duration_ms": final_scan["duration_ms"], + "orphan_detected_and_removed": cleaned["deleted_orphans"], + "missing_detected_and_restored": restored["restored_missing_objects"], + "fresh_orphan_preserved_until_grace": True, + "message_attachment_download_checksum": "unchanged", + "private_bucket": "anonymous request denied", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker.py b/student-tests/test_worker.py new file mode 100644 index 0000000..8e922fa --- /dev/null +++ b/student-tests/test_worker.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Local transformation checks for the lab5 image worker.""" +import hashlib +import io +import os +from pathlib import Path +import sys + +os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret") + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.worker import build_variants + + +def main(): + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment_id = "00000000-0000-4000-8000-000000000001" + variants = build_variants(attachment_id, original, pipeline_version=1) + by_name = {item["name"]: item for item in variants} + assert set(by_name) == {"thumbnail", "crop", "optimized"} + assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160) + assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128) + assert by_name["optimized"]["content_type"] == "image/webp" + for item in variants: + content = item["content"] + assert len(content) == item["size_bytes"] + assert hashlib.sha256(content).hexdigest() == item["sha256"] + with Image.open(io.BytesIO(content)) as image: + assert image.size == (item["width"], item["height"]) + newer = build_variants(attachment_id, original, pipeline_version=2) + assert all(f"/v2/" in item["object_key"] for item in newer) + assert all(f"/v1/" in item["object_key"] for item in variants) + assert by_name["thumbnail"]["sha256"] != { + item["name"]: item for item in newer + }["thumbnail"]["sha256"] + print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_worker_recovery_integration.py b/student-tests/test_worker_recovery_integration.py new file mode 100644 index 0000000..e2cb5fe --- /dev/null +++ b/student-tests/test_worker_recovery_integration.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Verify queued uploads and message links survive an API/worker restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify queued attachment recovery after API and worker restart") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + compose("stop", "worker") + worker_stopped = True + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted while worker was stopped") + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message") + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + 409, + actor=bob, + ) + compose("restart", "api") + compose("start", "worker") + worker_stopped = False + client.call("GET", "/health/ready", schema="Health") + ready = client.wait_ready(bob, attachment) + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(original_bytes == original, "recovered worker changed original bytes") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages", + actor=bob, + schema="MessagePage", + )["json"] + recovered_message = next(item for item in page["items"] if item["id"] == message["id"]) + check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link") + print( + json.dumps( + { + "result": "PASS", + "initial_status": attachment["status"], + "final_status": ready["status"], + "message_attachment_link_survived": True, + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + finally: + if worker_stopped: + compose("start", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1_benchmark.py b/student-tests/track1_benchmark.py new file mode 100644 index 0000000..fc637ad --- /dev/null +++ b/student-tests/track1_benchmark.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""Repeat a paced synthetic message workload with telemetry off and sampled.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import platform +import re +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + + +def command(args, capture=True): + result = subprocess.run( + args, + cwd=ROOT, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + return result.stdout.strip() if capture else "" + + +def set_env_value(text, name, value): + lines = text.splitlines() + prefix = name + "=" + for index, line in enumerate(lines): + if line.startswith(prefix): + lines[index] = prefix + value + break + else: + lines.append(prefix + value) + return "\n".join(lines) + "\n" + + +def parse_env(text): + values = {} + for line in text.splitlines(): + if line and not line.startswith("#") and "=" in line: + name, value = line.split("=", 1) + values[name] = value + return values + + +def wait_ready(url, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url + "/health/ready", timeout=2) as response: + if response.status == 200: + return + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(2) + raise RuntimeError("API did not become ready after the sampling mode change") + + +def parse_size(value): + match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value) + if not match: + return None + factors = { + "B": 1, + "kB": 1000, + "MB": 1000**2, + "GB": 1000**3, + "KiB": 1024, + "MiB": 1024**2, + "GiB": 1024**3, + } + return int(float(match.group(1)) * factors[match.group(2)]) + + +def docker_resources(): + rows = command( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}};{{.CPUPerc}};{{.MemUsage}}", + ] + ) + cpu_total = 0.0 + memory_total = 0 + found = False + for row in rows.splitlines(): + fields = row.split(";") + if len(fields) != 3 or not any( + name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-") + ): + continue + cpu = re.match(r"\s*([\d.]+)%", fields[1]) + memory = parse_size(fields[2].split("/", 1)[0]) + if cpu and memory is not None: + cpu_total += float(cpu.group(1)) + memory_total += memory + found = True + return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None + + +def storage_sizes(compose): + paths = { + "prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]), + "loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]), + "tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]), + } + result = {} + for name, args in paths.items(): + output = command(compose + args) + result[name] = int(output.split()[0]) + return result + + +def tempo_message_traces(compose, start, end): + url = ( + "http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000" + "&tags=service.name%%3Dmessenger-api" % (start, end) + ) + output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url]) + data = json.loads(output) + return [ + item + for item in data.get("traces", []) + if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages" + ] + + +def mean_spans(compose, traces, sample_limit=20): + counts = [] + for item in traces[:sample_limit]: + output = command( + compose + + [ + "exec", + "-T", + "tempo", + "wget", + "-qO-", + "http://127.0.0.1:3200/api/traces/" + item["traceID"], + ] + ) + data = json.loads(output) + count = sum( + len(scope.get("spans", [])) + for batch in data.get("batches", []) + for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", [])) + ) + counts.append(count) + return round(sum(counts) / len(counts), 2) if counts else 0.0 + + +def run_load(base_url, duration, concurrency, target_rps, alice, chat): + from client import Client, Failure + + barrier = threading.Barrier(concurrency + 1) + ticket_lock = threading.Lock() + ticket = 0 + latencies = [] + failures = 0 + results_lock = threading.Lock() + start_clock = time.monotonic() + start_epoch = int(time.time()) + deadline = start_clock + duration + + def worker(worker_id): + nonlocal ticket, failures + client = Client(base_url, timeout=5) + local_latencies = [] + local_failures = 0 + barrier.wait() + while True: + with ticket_lock: + current = ticket + ticket += 1 + due = start_clock + current / target_rps + if due >= deadline: + break + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + started = time.monotonic() + try: + client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current)) + local_latencies.append(time.monotonic() - started) + except Failure: + local_failures += 1 + with results_lock: + latencies.extend(local_latencies) + failures += local_failures + + samples = [] + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)] + barrier.wait() + while any(not future.done() for future in futures): + sample = docker_resources() + if sample is not None: + samples.append(sample) + time.sleep(2) + for future in futures: + future.result() + elapsed = time.monotonic() - start_clock + sorted_latencies = sorted(latencies) + + def percentile(value): + if not sorted_latencies: + return None + index = max(0, math.ceil(value * len(sorted_latencies)) - 1) + return round(sorted_latencies[index] * 1000, 3) + + total = len(sorted_latencies) + failures + report = { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": duration, + "target_rps": target_rps, + "elapsed_seconds": round(elapsed, 3), + "concurrency": concurrency, + "successful_requests": len(sorted_latencies), + "failed_requests": failures, + "error_ratio": round(failures / total, 6) if total else None, + "successful_rps": round(len(sorted_latencies) / elapsed, 3), + "successful_latency_ms": { + "p50": percentile(0.50), + "p95": percentile(0.95), + "p99": percentile(0.99), + }, + "docker_resource_samples": len(samples), + } + if samples: + report["docker_resources"] = { + "mean_cpu_percent_sum": round( + sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2 + ), + "peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples), + "peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples), + "services": ["api", "worker", "publisher", "otel-collector"], + } + return report, start_epoch, int(time.time()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", default=".env") + parser.add_argument("--base-url", default=None) + parser.add_argument("--duration", type=int, default=15) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--target-rps", type=int, default=20) + parser.add_argument("--warmup-seconds", type=int, default=5) + parser.add_argument("--output", default="evidence/track1-observability-cost.json") + args = parser.parse_args() + if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1: + parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive") + + env_path = ROOT / args.env_file + if not env_path.is_file(): + parser.error("create the local .env from .env.example and generate secrets first") + original = env_path.read_text(encoding="utf-8") + values = parse_env(original) + base_url = args.base_url or "http://localhost:" + values.get("APP_PORT", "8080") + compose = ["docker", "compose", "--env-file", str(env_path)] + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + results = [] + configurations = [ + ("off", "true", "1.0"), + ("sample-100-percent", "false", "1.0"), + ("sample-25-percent", "false", "0.25"), + ("sample-10-percent", "false", "0.1"), + ] + try: + for name, disabled, ratio in configurations: + changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled) + changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio") + changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio) + env_path.write_text(changed, encoding="utf-8") + command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"]) + wait_ready(base_url) + time.sleep(args.warmup_seconds) + from client import Client + + alice, _, _, chat = Client(base_url, timeout=10).bootstrap() + time.sleep(2) + before = storage_sizes(compose) + load, start, end = run_load(base_url, args.duration, args.concurrency, args.target_rps, alice, chat) + time.sleep(15) + after = storage_sizes(compose) + traces = tempo_message_traces(compose, start - 5, end + 20) + spans_per_trace = mean_spans(compose, traces) + results.append( + { + "configuration": name, + "sdk_disabled": disabled == "true", + "trace_sampling_ratio": float(ratio), + "load": load, + "window_unix_seconds": {"start": start, "end": end}, + "sampled_message_trace_count": len(traces), + "estimated_spans_per_message_trace": spans_per_trace, + "persistent_storage_bytes_delta": { + signal: after[signal] - before[signal] for signal in before + }, + } + ) + + baseline = next( + item["estimated_spans_per_message_trace"] + for item in results + if item["configuration"] == "sample-100-percent" + ) + for item in results: + expected_spans = item["load"]["successful_requests"] * baseline + observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"] + item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2) + + report = { + "experiment": "telemetry overhead under fixed-rate synthetic POST load", + "hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.", + "method": { + "duration_seconds": args.duration, + "concurrency": args.concurrency, + "target_rps": args.target_rps, + "warmup_seconds": args.warmup_seconds, + "telemetry_flush_wait_seconds": 15, + "hardware": { + "platform": platform.platform(), + "processor": platform.processor() or "unspecified", + "logical_cpus": os.cpu_count(), + }, + "sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.", + "storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.", + }, + "limitations": [ + "The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.", + "Database history grows between modes; all modes use identical duration and concurrency.", + "Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.", + "Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.", + "CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.", + ], + "results": results, + "balanced_configuration": "sample-25-percent", + } + output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + env_path.write_text(original, encoding="utf-8") + command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"]) + + +if __name__ == "__main__": + main() diff --git a/student-tests/upload_concurrency.py b/student-tests/upload_concurrency.py new file mode 100644 index 0000000..e45746c --- /dev/null +++ b/student-tests/upload_concurrency.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Run ten simultaneous near-6-MiB uploads and sample API container memory.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +import re +import subprocess +import threading +import time +import uuid +from io import BytesIO +from pathlib import Path +import sys + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} +USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/") + + +def usage_bytes(line): + match = USAGE_RE.search(line) + return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0 + + +def api_containers(): + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + ids = result.stdout.split() + check(len(ids) >= 2, "at least two running API containers are required") + return ids + + +def sample_once(ids): + output = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + values = [usage_bytes(line) for line in output] + check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage") + return max(values) + + +def main(): + args = arguments("Bounded-memory ten-client upload experiment") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + width = height = 1400 + content = BytesIO() + Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save( + content, format="PNG", compress_level=0 + ) + payload = content.getvalue() + check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit") + ids = api_containers() + baseline = sample_once(ids) + samples = [] + stats_process = subprocess.Popen( + ["docker", "stats", "--format", "{{.MemUsage}}", *ids], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def sampler(): + if stats_process.stdout is None: + return + for line in stats_process.stdout: + value = usage_bytes(line) + if value: + samples.append(value) + + sampling_thread = threading.Thread(target=sampler, daemon=True) + sampling_thread.start() + count = 10 + barrier = threading.Barrier(count) + + def upload(index): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=15) + return worker.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png", + 202, + actor=alice, + raw=payload, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + started = time.monotonic() + try: + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(upload, range(count))) + finally: + stopped_by_test = False + if stats_process.poll() is None: + stats_process.kill() + stopped_by_test = True + try: + stats_process.wait(timeout=5) + except subprocess.TimeoutExpired: + stats_process.kill() + stats_process.wait(timeout=5) + sampling_thread.join(timeout=5) + stats_error = stats_process.stderr.read() if stats_process.stderr is not None else "" + check( + stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9), + "docker stats sampling process failed (exit " + + str(stats_process.returncode) + + "): " + + stats_error.strip()[:300], + ) + check(bool(samples), "docker stats produced no memory samples") + peak = max([baseline] + samples) + check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed") + print( + json.dumps( + { + "uploads": count, + "concurrency": count, + "bytes_per_upload": len(payload), + "duration_seconds": round(time.monotonic() - started, 3), + "api_containers": len(ids), + "sample_count": len(samples) // len(ids), + "baseline_max_container_bytes": baseline, + "peak_max_container_bytes": peak, + "peak_delta_bytes": max(0, peak - baseline), + "sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence", + "memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) -- 2.54.0 From cc87ce7e90ae4f16a30b0771186106697c537a23 Mon Sep 17 00:00:00 2001 From: kaktuz20000 <149298757+kaktuz20000@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:04:37 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D1=84=D0=B8=D0=BA=D1=81=20=D0=B3=D0=B8?= =?UTF-8?q?=D1=81=D1=82=D0=BE=D0=B3=D1=80=D0=B0=D0=BC=D0=BC=20=D1=81=20?= =?UTF-8?q?=D0=BA=D1=80=D1=83=D0=BF=D0=BD=D1=8B=D0=BC=D0=B8=20=D0=B8=D0=BD?= =?UTF-8?q?=D1=82=D0=B5=D1=80=D0=B2=D0=B0=D0=BB=D0=B0=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra/otel/README.md | 2 ++ src/telemetry.py | 27 ++++++++++++++++++++++++++- src/worker.py | 15 +++++++++++---- student-tests/test_worker.py | 15 +++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/infra/otel/README.md b/infra/otel/README.md index 386510d..58bf90b 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -7,3 +7,5 @@ The Compose stack pins Collector Contrib 0.163.2, Prometheus 2.55.1, Loki 3.3.2, Loki, Tempo, and Prometheus retain local data in named volumes. Grafana datasources, the dashboard, and Prometheus alert rules are provisioned from this directory. Prometheus evaluates the 7-day rolling API availability SLI and local alerts; use the alert annotations and `REPORT.md` runbooks to investigate. API, publisher, and worker share a W3C trace context through the database outbox and the broker payload. Application logs contain only structured metadata, trace/span IDs, request IDs, route templates, and sanitized event names. Message bodies, credentials, filenames, and signed URLs are not exported. + +HTTP request duration, worker job duration, and attachment time-to-ready histograms use explicit second-based buckets. HTTP buckets are dense from 1 ms through 100 ms to resolve normal API latency; job buckets extend to 60 s and end-to-end readiness buckets to 600 s. This keeps Prometheus percentile interpolation useful across all three time scales. diff --git a/src/telemetry.py b/src/telemetry.py index 9fb6572..9d1a244 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -18,6 +18,7 @@ from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -36,6 +37,21 @@ _runtime = { "cpu": 0.0, "memory": 0, } +HISTOGRAM_BUCKETS = { + "messenger.http.duration": ( + 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.0075, 0.008, + 0.0085, 0.009, 0.0095, 0.01, 0.0125, 0.015, 0.02, 0.025, + 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + ), + "messenger.jobs.duration": ( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1, 2.5, 5, 10, 30, 60, + ), + "messenger.attachments.time_to_ready": ( + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, + 5, 10, 30, 60, 120, 300, 600, + ), +} _provider = None _meter = None _service_name = "messenger" @@ -126,7 +142,16 @@ def initialize(service_name: str) -> None: export_interval_millis=10000, export_timeout_millis=5000, ) - metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[reader])) + views = [ + View( + instrument_name=name, + aggregation=ExplicitBucketHistogramAggregation(boundaries=buckets), + ) + for name, buckets in HISTOGRAM_BUCKETS.items() + ] + metrics.set_meter_provider( + MeterProvider(resource=resource, metric_readers=[reader], views=views) + ) logger_provider = LoggerProvider(resource=resource) logger_provider.add_log_record_processor( BatchLogRecordProcessor( diff --git a/src/worker.py b/src/worker.py index 0828d6c..8a0b2b3 100644 --- a/src/worker.py +++ b/src/worker.py @@ -83,6 +83,15 @@ def _encode(image: Image.Image, image_format: str, **options) -> bytes: return output.getvalue() +def _resize_within(image: Image.Image, bounds: tuple[int, int]) -> Image.Image: + width, height = image.size + scale = min(1.0, bounds[0] / width, bounds[1] / height) + size = (max(1, int(width * scale)), max(1, int(height * scale))) + if size == image.size: + return image.copy() + return image.resize(size, Image.Resampling.LANCZOS) + + def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1): quality = PIPELINE_QUALITY.get(pipeline_version) if quality is None: @@ -90,9 +99,8 @@ def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1 image = _rgb_image(content) try: variants = [] - thumbnail = image.copy() + thumbnail = _resize_within(image, (256, 256)) try: - thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS) thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True) variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes)) finally: @@ -105,9 +113,8 @@ def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1 finally: crop.close() - optimized = image.copy() + optimized = _resize_within(image, (1280, 1280)) try: - optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6) variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes)) finally: diff --git a/student-tests/test_worker.py b/student-tests/test_worker.py index 8e922fa..6e7b00d 100644 --- a/student-tests/test_worker.py +++ b/student-tests/test_worker.py @@ -38,6 +38,21 @@ def main(): assert by_name["thumbnail"]["sha256"] != { item["name"]: item for item in newer }["thumbnail"]["sha256"] + + odd_ratio = io.BytesIO() + Image.new("RGB", (319, 200), "blue").save(odd_ratio, format="PNG") + odd_variants = { + item["name"]: item for item in build_variants(attachment_id, odd_ratio.getvalue()) + } + assert (odd_variants["thumbnail"]["width"], odd_variants["thumbnail"]["height"]) == (256, 160) + + small_image = io.BytesIO() + Image.new("RGB", (100, 50), "red").save(small_image, format="PNG") + small_variants = { + item["name"]: item for item in build_variants(attachment_id, small_image.getvalue()) + } + assert (small_variants["thumbnail"]["width"], small_variants["thumbnail"]["height"]) == (100, 50) + assert (small_variants["optimized"]["width"], small_variants["optimized"]["height"]) == (100, 50) print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums") -- 2.54.0 From c24b39eeb43efb1971296800a6381612c589f9cf Mon Sep 17 00:00:00 2001 From: kaktuz20000 <149298757+kaktuz20000@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:10:05 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B2=D0=BE=20=D0=B2=D1=80=D0=B5?= =?UTF-8?q?=D0=BC=D1=8F=20=D0=BB=D0=B0=D0=B1=D1=8B=20=D0=BE=D1=88=D0=B8?= =?UTF-8?q?=D0=B1=D0=BA=D0=B8=20=D0=B3=D1=80=D0=B0=D1=84=D0=B0=D0=BD=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 29 +++-- evidence/track1-observability-cost.json | 108 +++++++++---------- infra/otel/README.md | 12 +-- infra/otel/grafana/dashboards/messenger.json | 4 +- 4 files changed, 74 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 465d80d..a607238 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку. -В этой реализации сохранён сервер Lab 5 и добавлены OpenTelemetry-инструментация, Collector, Prometheus, Loki, Tempo, Grafana, alert rules и эксперимент по sampling. Язык и framework свободные; UI не обязателен. +Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) ## Откуда начинаем -Бизнес-API и фоновый pipeline перенесены из lab5; контракт не меняется. Стенд сохраняет API-реплики, PostgreSQL, S3, RabbitMQ, publisher, worker и reconciler. +Перенесите минимум lab5. Бизнес-API не меняется. Должны оставаться API-реплики, БД, S3, broker и worker. **Архитектура:** API/worker → OTLP → OTel Collector → хранилища metrics/logs/traces → Grafana; queue context связывает API и worker. @@ -47,24 +47,23 @@ ## Как начать и проверить ```sh -# Локальный offline check, Python 3.9+: +# Работает прямо в skeleton, Python 3.9+; приложение не запускается: make check -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -cp .env.example .env -scripts/bootstrap-secrets.sh -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь health/ready. По умолчанию API публикуется на localhost:8080. -make test PYTHON=.venv/bin/python BASE_URL=http://localhost:8080 +# После реализации запустите свой сервер/стенд по REPORT.md. +# По умолчанию тест использует http://localhost:8080. +make test ``` -`.env.example` содержит локальные настройки без секретов. Секреты генерируются в `secrets/` и не должны коммититься. `compose.example.yaml` — worksheet с пустым `services`, реальный запускаемый стенд — `compose.yaml`. Образы закреплены версиями, данные приложения и трёх backend хранятся в named volumes. +`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. -Для полной OpenAPI-валидации установите `requirements-dev.txt` и выполните `make validate PYTHON=.venv/bin/python`. Основные локальные проверки pipeline: `make test-worker`, `make test-reconcile`, `make test-pipeline-faults`. Сводный отчёт о затратах sampling запускается после старта стенда командой `make benchmark-observability PYTHON=.venv/bin/python BASE_URL=http://localhost:8080`. - -Grafana доступна на `http://localhost:3000`; пароль администратора находится в локальном `secrets/grafana_admin_password`. OTLP endpoints и telemetry backends доступны только внутри Compose-сети. При занятом порте задайте другой `APP_PORT` в локальном `.env` и соответствующий `BASE_URL`. +```sh +# Compose-вариант после реализации: +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождитесь /health/ready; затем make test. +# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +``` Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): diff --git a/evidence/track1-observability-cost.json b/evidence/track1-observability-cost.json index 8a14325..63dd98f 100644 --- a/evidence/track1-observability-cost.json +++ b/evidence/track1-observability-cost.json @@ -8,7 +8,7 @@ "warmup_seconds": 3, "telemetry_flush_wait_seconds": 15, "hardware": { - "platform": "macOS-27.0.1-arm64-arm-64bit-Mach-O", + "platform": "macOS-27.2.1-arm64-arm-64bit-Mach-O", "processor": "arm", "logical_cpus": 10 }, @@ -31,22 +31,22 @@ "workload": "paced synthetic POST messages; no automatic retries", "requested_duration_seconds": 15, "target_rps": 20, - "elapsed_seconds": 16.901, + "elapsed_seconds": 17.135, "concurrency": 2, "successful_requests": 300, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 17.75, + "successful_rps": 17.508, "successful_latency_ms": { - "p50": 7.449, - "p95": 12.198, - "p99": 14.054 + "p50": 8.449, + "p95": 15.118, + "p99": 21.618 }, "docker_resource_samples": 4, "docker_resources": { - "mean_cpu_percent_sum": 17.18, - "peak_cpu_percent_sum": 46.29, - "peak_memory_bytes_sum": 331140298, + "mean_cpu_percent_sum": 21.28, + "peak_cpu_percent_sum": 38.75, + "peak_memory_bytes_sum": 567678072, "services": [ "api", "worker", @@ -56,15 +56,15 @@ } }, "window_unix_seconds": { - "start": 1793781279, - "end": 1793781296 + "start": 1794559044, + "end": 1794559061 }, "sampled_message_trace_count": 0, "estimated_spans_per_message_trace": 0.0, "persistent_storage_bytes_delta": { - "prometheus": 25354, + "prometheus": 1492625, "loki": 0, - "tempo": 122589 + "tempo": 0 }, "estimated_spans_not_recorded": 2400.0 }, @@ -76,22 +76,22 @@ "workload": "paced synthetic POST messages; no automatic retries", "requested_duration_seconds": 15, "target_rps": 20, - "elapsed_seconds": 17.678, + "elapsed_seconds": 18.348, "concurrency": 2, "successful_requests": 300, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 16.97, + "successful_rps": 16.35, "successful_latency_ms": { - "p50": 8.22, - "p95": 14.766, - "p99": 37.456 + "p50": 8.284, + "p95": 15.915, + "p99": 18.98 }, "docker_resource_samples": 4, "docker_resources": { - "mean_cpu_percent_sum": 25.32, - "peak_cpu_percent_sum": 49.25, - "peak_memory_bytes_sum": 340199994, + "mean_cpu_percent_sum": 31.85, + "peak_cpu_percent_sum": 41.59, + "peak_memory_bytes_sum": 550523370, "services": [ "api", "worker", @@ -101,15 +101,15 @@ } }, "window_unix_seconds": { - "start": 1793781337, - "end": 1793781354 + "start": 1794559100, + "end": 1794559118 }, "sampled_message_trace_count": 300, "estimated_spans_per_message_trace": 8.0, "persistent_storage_bytes_delta": { - "prometheus": 27422, - "loki": 240231, - "tempo": 428182 + "prometheus": 36951, + "loki": 204007, + "tempo": 422028 }, "estimated_spans_not_recorded": 0.0 }, @@ -121,22 +121,22 @@ "workload": "paced synthetic POST messages; no automatic retries", "requested_duration_seconds": 15, "target_rps": 20, - "elapsed_seconds": 16.811, + "elapsed_seconds": 17.76, "concurrency": 2, "successful_requests": 300, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 17.845, + "successful_rps": 16.891, "successful_latency_ms": { - "p50": 6.822, - "p95": 12.894, - "p99": 14.415 + "p50": 7.793, + "p95": 14.838, + "p99": 16.862 }, "docker_resource_samples": 4, "docker_resources": { - "mean_cpu_percent_sum": 26.23, - "peak_cpu_percent_sum": 47.71, - "peak_memory_bytes_sum": 352195705, + "mean_cpu_percent_sum": 15.26, + "peak_cpu_percent_sum": 36.45, + "peak_memory_bytes_sum": 550554827, "services": [ "api", "worker", @@ -146,15 +146,15 @@ } }, "window_unix_seconds": { - "start": 1793781394, - "end": 1793781411 + "start": 1794559157, + "end": 1794559175 }, "sampled_message_trace_count": 88, "estimated_spans_per_message_trace": 8.0, "persistent_storage_bytes_delta": { - "prometheus": 27362, - "loki": -7107212, - "tempo": 4092487 + "prometheus": 40456, + "loki": 269949, + "tempo": 1026146 }, "estimated_spans_not_recorded": 1696.0 }, @@ -166,22 +166,22 @@ "workload": "paced synthetic POST messages; no automatic retries", "requested_duration_seconds": 15, "target_rps": 20, - "elapsed_seconds": 17.248, + "elapsed_seconds": 18.355, "concurrency": 2, "successful_requests": 300, "failed_requests": 0, "error_ratio": 0.0, - "successful_rps": 17.393, + "successful_rps": 16.344, "successful_latency_ms": { - "p50": 7.351, - "p95": 13.809, - "p99": 17.187 + "p50": 7.625, + "p95": 14.84, + "p99": 18.012 }, "docker_resource_samples": 4, "docker_resources": { - "mean_cpu_percent_sum": 28.82, - "peak_cpu_percent_sum": 49.75, - "peak_memory_bytes_sum": 339979794, + "mean_cpu_percent_sum": 22.53, + "peak_cpu_percent_sum": 39.51, + "peak_memory_bytes_sum": 550198311, "services": [ "api", "worker", @@ -191,17 +191,17 @@ } }, "window_unix_seconds": { - "start": 1793781454, - "end": 1793781471 + "start": 1794559215, + "end": 1794559233 }, - "sampled_message_trace_count": 25, + "sampled_message_trace_count": 38, "estimated_spans_per_message_trace": 8.0, "persistent_storage_bytes_delta": { - "prometheus": 31886, - "loki": 239322, - "tempo": 106458 + "prometheus": 42617, + "loki": 240135, + "tempo": 1924562 }, - "estimated_spans_not_recorded": 2200.0 + "estimated_spans_not_recorded": 2096.0 } ], "balanced_configuration": "sample-25-percent" diff --git a/infra/otel/README.md b/infra/otel/README.md index 58bf90b..c46fe89 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,11 +1,7 @@ -# OpenTelemetry deployment +# Конфигурация наблюдаемости — реализовать -The Compose stack pins Collector Contrib 0.163.2, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.7.1, and Grafana 11.4.0. OTLP gRPC/HTTP listeners and backend query ports are private to the Compose network; only the API load balancer and Grafana UI bind to loopback. +Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. -`collector.yaml` receives all three OTLP signals. Metrics are exposed to Prometheus on the private Collector endpoint; logs are exported to Loki's OTLP receiver and traces to Tempo. Memory limiter and batch processors bound Collector memory. Trace/log exporter queues are capped at 256 requests and retry for at most 30 seconds; SDK span/log queues are capped at 4096 records with batches of at most 512. On backend outage, telemetry is dropped after bounded retries rather than blocking API work indefinitely. Metrics are pull-exported and can be missed while Prometheus is unavailable. +Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. -Loki, Tempo, and Prometheus retain local data in named volumes. Grafana datasources, the dashboard, and Prometheus alert rules are provisioned from this directory. Prometheus evaluates the 7-day rolling API availability SLI and local alerts; use the alert annotations and `REPORT.md` runbooks to investigate. - -API, publisher, and worker share a W3C trace context through the database outbox and the broker payload. Application logs contain only structured metadata, trace/span IDs, request IDs, route templates, and sanitized event names. Message bodies, credentials, filenames, and signed URLs are not exported. - -HTTP request duration, worker job duration, and attachment time-to-ready histograms use explicit second-based buckets. HTTP buckets are dense from 1 ms through 100 ms to resolve normal API latency; job buckets extend to 60 s and end-to-end readiness buckets to 600 s. This keeps Prometheus percentile interpolation useful across all three time scales. +Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json index 04e026b..7e37518 100644 --- a/infra/otel/grafana/dashboards/messenger.json +++ b/infra/otel/grafana/dashboards/messenger.json @@ -119,7 +119,7 @@ "gridPos": {"h": 7, "w": 8, "x": 16, "y": 22}, "id": 10, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, - "targets": [{"expr": "max(messenger_process_memory_usage)", "legendFormat": "RSS", "refId": "A"}], + "targets": [{"expr": "max(messenger_process_memory_usage_bytes)", "legendFormat": "RSS", "refId": "A"}], "title": "Process memory", "type": "timeseries" }, @@ -129,7 +129,7 @@ "gridPos": {"h": 7, "w": 8, "x": 0, "y": 29}, "id": 11, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, - "targets": [{"expr": "max(messenger_process_cpu_utilization)", "legendFormat": "CPU / core", "refId": "A"}], + "targets": [{"expr": "max(messenger_process_cpu_utilization_ratio)", "legendFormat": "CPU / core", "refId": "A"}], "title": "Process CPU utilization", "type": "timeseries" }, -- 2.54.0