From 5a6fcff3ef1e91788d64ebb5bd2fae07640e579e Mon Sep 17 00:00:00 2001 From: kaktuz20000 <149298757+kaktuz20000@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:22:35 +0300 Subject: [PATCH] =?UTF-8?q?=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 51 +- Dockerfile | 22 + MAINTAINERS.md | 6 +- Makefile | 66 +- README.md | 44 +- REPORT.md | 105 +- compose.yaml | 871 ++++++++++++++++ contracts/trust-matrix.md | 54 +- evidence/argon2-budget.json | 1 + evidence/dependency-audit.json | 1 + evidence/failover.json | 7 + evidence/leaf-rotation.json | 1 + evidence/persistence.json | 7 + evidence/pipeline-faults.json | 58 ++ evidence/telemetry-redaction.json | 1 + evidence/worker-isolation.json | 1 + evidence/worker-rotation.json | 1 + haproxy.cfg | 38 + infra/README.md | 2 +- infra/openbao/README.md | 15 +- infra/openbao/api-policy.hcl | 3 + infra/openbao/config.hcl | 15 + infra/openbao/worker-policy.hcl | 3 + infra/otel/alerts.yml | 42 + infra/otel/collector.yaml | 80 ++ infra/otel/grafana/dashboards/messenger.json | 174 ++++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/datasources.yml | 39 + infra/otel/loki.yml | 37 + infra/otel/prometheus-web.yml | 3 + infra/otel/prometheus.yml | 20 + infra/otel/tempo.yml | 36 + infra/postgres/pg_hba.conf | 3 + migrations/001_initial.sql | 43 + migrations/002_idempotency.sql | 16 + migrations/003_attachments.sql | 59 ++ migrations/004_pipeline_replay.sql | 16 + migrations/005_observability_context.sql | 3 + migrations/006_login_rate_limits.sql | 7 + requirements-dev.txt | 3 +- requirements.txt | 13 + scripts/bootstrap-openbao.py | 130 +++ scripts/bootstrap-pki.py | 201 ++++ scripts/bootstrap-secrets.py | 254 +++++ scripts/issue-service-pki.py | 92 ++ scripts/prepare-secure-secrets.py | 228 +++++ scripts/reconcile-loop.sh | 33 + scripts/restart.sh | 28 + scripts/stop-one.sh | 20 + security/THREAT_MODEL.md | 26 +- src/__init__.py | 1 + src/app.py | 77 ++ src/bootstrap_s3.py | 111 +++ src/config.py | 126 +++ src/errors.py | 26 + src/handlers.py | 653 ++++++++++++ src/jobs.py | 33 + src/migrate.py | 27 + src/object_store.py | 101 ++ src/pipeline_versions.py | 4 + src/publisher.py | 100 ++ src/reconcile.py | 128 +++ src/replay_attachment.py | 34 + src/security.py | 134 +++ src/server.py | 13 + src/store.py | 934 ++++++++++++++++++ src/telemetry.py | 300 ++++++ src/worker.py | 383 +++++++ student-tests/test_attachments.py | 98 ++ student-tests/test_idempotency_race.py | 47 + .../test_outbox_recovery_integration.py | 66 ++ student-tests/test_pipeline_faults.py | 485 +++++++++ .../test_pipeline_replay_integration.py | 167 ++++ student-tests/test_reconcile.py | 102 ++ student-tests/test_reconcile_integration.py | 211 ++++ student-tests/test_worker.py | 45 + .../test_worker_recovery_integration.py | 78 ++ student-tests/track1_benchmark.py | 365 +++++++ student-tests/upload_concurrency.py | 151 +++ tests/README.md | 37 +- tests/argon2_budget.py | 93 ++ tests/internal_mtls.py | 45 + tests/leaf_certificate_rotation.py | 207 ++++ tests/login_rate_limit.py | 39 + tests/openbao_acl.py | 95 ++ tests/resilience.py | 2 +- tests/telemetry_redaction.py | 116 +++ tests/test_security_primitives.py | 54 + tests/worker_identity_rotation.py | 168 ++++ tests/worker_isolation.py | 132 +++ 90 files changed, 8848 insertions(+), 130 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/argon2-budget.json create mode 100644 evidence/dependency-audit.json create mode 100644 evidence/failover.json create mode 100644 evidence/leaf-rotation.json create mode 100644 evidence/persistence.json create mode 100644 evidence/pipeline-faults.json create mode 100644 evidence/telemetry-redaction.json create mode 100644 evidence/worker-isolation.json create mode 100644 evidence/worker-rotation.json create mode 100644 haproxy.cfg create mode 100644 infra/openbao/api-policy.hcl create mode 100644 infra/openbao/config.hcl create mode 100644 infra/openbao/worker-policy.hcl create mode 100644 infra/otel/alerts.yml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/grafana/dashboards/messenger.json create mode 100644 infra/otel/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/otel/grafana/provisioning/datasources/datasources.yml create mode 100644 infra/otel/loki.yml create mode 100644 infra/otel/prometheus-web.yml create mode 100644 infra/otel/prometheus.yml create mode 100644 infra/otel/tempo.yml create mode 100644 infra/postgres/pg_hba.conf create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_pipeline_replay.sql create mode 100644 migrations/005_observability_context.sql create mode 100644 migrations/006_login_rate_limits.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap-openbao.py create mode 100644 scripts/bootstrap-pki.py create mode 100644 scripts/bootstrap-secrets.py create mode 100644 scripts/issue-service-pki.py create mode 100644 scripts/prepare-secure-secrets.py create mode 100755 scripts/reconcile-loop.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/config.py create mode 100644 src/errors.py create mode 100644 src/handlers.py create mode 100644 src/jobs.py create mode 100644 src/migrate.py create mode 100644 src/object_store.py create mode 100644 src/pipeline_versions.py create mode 100644 src/publisher.py create mode 100644 src/reconcile.py create mode 100644 src/replay_attachment.py create mode 100644 src/security.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 src/telemetry.py create mode 100644 src/worker.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_idempotency_race.py create mode 100644 student-tests/test_outbox_recovery_integration.py create mode 100644 student-tests/test_pipeline_faults.py create mode 100644 student-tests/test_pipeline_replay_integration.py create mode 100644 student-tests/test_reconcile.py create mode 100644 student-tests/test_reconcile_integration.py create mode 100644 student-tests/test_worker.py create mode 100644 student-tests/test_worker_recovery_integration.py create mode 100644 student-tests/track1_benchmark.py create mode 100644 student-tests/upload_concurrency.py create mode 100644 tests/argon2_budget.py create mode 100644 tests/internal_mtls.py create mode 100644 tests/leaf_certificate_rotation.py create mode 100644 tests/login_rate_limit.py create mode 100644 tests/openbao_acl.py create mode 100644 tests/telemetry_redaction.py create mode 100644 tests/test_security_primitives.py create mode 100644 tests/worker_identity_rotation.py create mode 100644 tests/worker_isolation.py diff --git a/.env.example b/.env.example index 2e9f054..eef75e5 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,18 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Local Compose settings only. Credentials and private keys live under ignored secrets/. +APP_PORT=8443 API_REPLICAS=2 +API_CPU_LIMIT=1.0 +API_MEMORY_LIMIT=512M +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=https://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +MAX_CONCURRENT_UPLOADS=10 +WORKER_CONCURRENCY=1 +WORKER_REPLICAS=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab -# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. -PUBLIC_BASE_URL=https://localhost:8443 -TLS_CA_FILE=/run/secrets/ca.crt -TLS_CERT_FILE=/run/secrets/service.crt -TLS_KEY_FILE=/run/secrets/service.key -BAO_ADDR=https://openbao:8200 -BAO_CACERT=/run/secrets/bao-ca.crt -# No root token here. Bootstrap operator -> short-lived service auth -> scoped token. -ARGON2_MEMORY_KIB=19456 -ARGON2_ITERATIONS=2 -ARGON2_PARALLELISM=1 +RECONCILE_INTERVAL_SECONDS=900 +RECONCILE_GRACE_SECONDS=3600 +GRAFANA_PORT=3000 +OTEL_SDK_DISABLED=false +OTEL_TRACES_SAMPLER=parentbased_traceidratio +OTEL_TRACES_SAMPLER_ARG=1.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..be99144 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +CMD ["python3", "src/server.py"] diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d3b18f7..1dc844c 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -31,8 +31,8 @@ Исходные README/контракт/публичные tests сохраняются, реализация переносится студентом между лабами через собственные исходники и миграции. Исправление общей спецификации публикуйте отдельным изменением, объясняя влияние на все затронутые лабораторные. -## Проверка этой версии skeleton +## Проверка текущей реализации -На 2026-09-10 выполнены структурные проверки и полная валидация всех семи OpenAPI/JSON Schema, проверка локальных Markdown-ссылок/YAML и декодирование PNG fixture. HTTP-клиенты проверены на временном изолированном тестовом стенде: позитивный smoke 1–7, отрицательные варианты неправильного sender, pagination, ACL, logout, bytes и пустого API; также пути persistence/failover/load и внешнего TLS. +На 2026-11-04 выполнены `make check`, полная OpenAPI-валидация, тесты парольных примитивов, проверка Python 3.9 syntax grammar, `docker compose config --quiet` и сборка API image. Реальный `MutualTLSServer` проверен с локальными сертификатами: запрос без client certificate, с foreign CA и с неразрешённым URI SAN отвергнуты; разрешённая LB identity принята. -Временный стенд не включён в студенческие каталоги и не является эталонным решением. Реальное приложение, Compose, PostgreSQL/S3/broker и mTLS/OpenBao в skeleton отсутствуют: их работоспособность не заявляется. Docker CLI в среде подготовки не был доступен. Это проверка материалов и тестовых драйверов, не приёмка будущих решений студентов. +Полный Compose smoke, OpenBao runtime allow/deny, persistence/failover и нагрузочный прогон следует выполнять на изолированном стенде после operator init/unseal и создания локальных credentials. diff --git a/Makefile b/Makefile index 5ad43ad..f6394c1 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-security +.PHONY: help check validate test test-primitives test-internal-mtls test-login-limit test-openbao-acl test-worker-isolation test-worker-rotation test-leaf-rotation test-telemetry-redaction test-argon-budget load benchmark-observability \ + test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults \ + test-pipeline-replay test-outbox-recovery test-worker-recovery test-reconcile-integration \ + test-upload-concurrency test-persistence test-failover test-security help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -18,9 +21,70 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-primitives: + $(PYTHON) -m unittest discover -s tests -p 'test_*.py' + +test-internal-mtls: + docker compose exec -T api python3 /app/tests/internal_mtls.py + +test-login-limit: + $(PYTHON) tests/login_rate_limit.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-openbao-acl: + $(PYTHON) tests/openbao_acl.py --ca-file "$(CURDIR)/secrets/pki/ca.crt" + +test-worker-isolation: + docker compose exec -T worker python3 /app/tests/worker_isolation.py + +test-worker-rotation: + @test -n "$(ROOT_TOKEN_FILE)" || (echo "Set ROOT_TOKEN_FILE to an operator-only root-token file"; exit 2) + $(PYTHON) tests/worker_identity_rotation.py --ca-file "$(CURDIR)/secrets/pki/ca.crt" --root-token-file "$(ROOT_TOKEN_FILE)" + +test-leaf-rotation: + $(PYTHON) tests/leaf_certificate_rotation.py + +test-telemetry-redaction: + $(PYTHON) tests/telemetry_redaction.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-argon-budget: + docker compose exec -T api python3 /app/tests/argon2_budget.py + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +benchmark-observability: + $(PYTHON) student-tests/track1_benchmark.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency-race: + $(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + $(PYTHON) student-tests/test_reconcile.py + +test-worker: + $(PYTHON) student-tests/test_worker.py + +test-pipeline-faults: + $(PYTHON) student-tests/test_pipeline_faults.py + +test-pipeline-replay: + $(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-outbox-recovery: + $(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-worker-recovery: + $(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile-integration: + $(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-upload-concurrency: + $(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/README.md b/README.md index b329f91..e4da09c 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ Защитить работающий мессенджер на границах доверия и показать защиту отрицательными проверками: чужой доступ, неверный сертификат, недостаточные права и отзыв credentials. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Основа приложения и проверок перенесена из lab6. В этой работе сохранены REST API, PostgreSQL, S3-вложения, RabbitMQ worker и OpenTelemetry-стенд; внешний ingress работает по HTTPS, связь LB→API — по mTLS с проверкой URI SAN, пароли хешируются Argon2id, а рабочие secrets выдаются сервисам через OpenBao AppRole. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) -## Откуда начинаем +## Состав стенда -Перенесите минимум lab6. Бизнес-API сохраняется, внешний endpoint теперь HTTPS. Новые секреты и ключи создаются для стенда, не поставляются в skeleton. - -**Архитектура:** Клиент — TLS → LB — mTLS → API; API/worker — TLS (+ клиентские cert где требуется матрицей) → зависимости; OpenBao выдаёт scoped credentials. Grafana/OTel остаются внутри доверенной сети. +**Архитектура:** клиент — TLS → HAProxy — mTLS → API; API, worker, publisher, migrator и reconciler получают секреты через отдельные AppRole policies. Основные сервисы и observability сохраняют архитектуру lab6. ## Что сделать @@ -50,33 +48,41 @@ ## Как начать и проверить ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +cp .env.example .env +.venv/bin/python scripts/bootstrap-secrets.py +.venv/bin/python scripts/bootstrap-pki.py make check -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует https://localhost:8443. -make test -``` +# Сначала хранилища и OpenBao: +docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao +# Оператор инициализирует и unseal-ит OpenBao; root token/unseal shares +# сохраняются вне репозитория и передаются только скрипту bootstrap: +docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init +docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal +# Повторите unseal для требуемого порога shares. +python3 scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. - -```sh -# Compose-вариант после реализации: docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-worker-isolation +make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token ``` -Для HTTPS передайте публичный CA: `make test CA_FILE=/absolute/path/to/ca.crt`. CA certificate можно распространять; private keys создаются локально. `make test-security` проверяет только внешний TLS trust; внутренние отрицательные тесты реализуйте по [trust matrix](contracts/trust-matrix.md). +`.env.example` содержит открытые локальные настройки. PKI и локальные bootstrap secrets создаются в игнорируемом Git каталоге `secrets/`. `compose.example.yaml` — worksheet; запускаемый стенд находится в `compose.yaml`. Образы закреплены версиями, данные приложения и OpenBao хранятся в named volumes. Обычная остановка сохраняет учебные данные (`docker compose down`); намеренное удаление данных выполняется отдельно (`docker compose down -v`). Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): ```sh -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt make validate PYTHON=.venv/bin/python +make test-primitives PYTHON=.venv/bin/python +make test-openbao-acl PYTHON=.venv/bin/python +make test-argon-budget +.venv/bin/pip-audit -r requirements.txt ``` Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. diff --git a/REPORT.md b/REPORT.md index 9801db4..c257311 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,110 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: в исходном отчёте lab6 не указаны. +- Целевая оценка: 5; выбранный трек: 1 (компрометация worker). Ниже указаны повторяемая методика и результаты на локальном Compose-стенде. +- Commit/PR: не создан. +- Использованные источники, библиотеки и помощь ИИ: собственная реализация lab6; README/COURSE и контракты lab7; документация Argon2, Python `ssl`, OpenBao AppRole/ACL и Docker Compose. Использованы psycopg, boto3, Pillow, pika, OpenTelemetry и argon2-cffi (версии закреплены в `requirements.txt`). ИИ использовался для переноса, адаптации и проверки кода. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose v2, OpenSSL и Python 3.11+. Образы зафиксированы в `compose.yaml`; приложение собирается из `python:3.12.7-slim`. На машине разработки использован Python 3.13.2. + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +cp .env.example .env +.venv/bin/python scripts/bootstrap-secrets.py +.venv/bin/python scripts/bootstrap-pki.py +docker compose --env-file .env config --quiet + +# Поднять зависимости и OpenBao до приложения. +docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao +# Оператор выполняет init и unseal; shares и root token хранятся вне репозитория. +docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init +docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal +# Повторить unseal до достижения порога, затем: +.venv/bin/python scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token + +docker compose --env-file .env up -d --build +make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-primitives PYTHON=.venv/bin/python +make test-internal-mtls +make test-login-limit PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +``` + +Для остановки без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных и OpenBao state: `docker compose --env-file .env down -v`. Скрипты создают credentials и ключи только локально в исключённом из Git каталоге `secrets/`; не копируйте его в репозиторий. Секреты и unseal material не помещать в командную строку, отчёт или evidence. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Перенесены приложение, миграции, обработка вложений и тестовые сценарии lab6. PostgreSQL хранит пользователей, чаты, членство, сообщения, сессии, idempotency state и metadata вложений. SeaweedFS хранит оригиналы и variants; RabbitMQ обеспечивает at-least-once обработку через transactional outbox, отдельный publisher и worker; reconciler сверяет состояние. OTel Collector направляет сигналы в Prometheus, Loki и Tempo, Grafana загружает provisioning из репозитория. + +HAProxy завершает внешний TLS на `localhost:8443`; сертификат ingress содержит SAN для `localhost` и `127.0.0.1`. HAProxy проверяет CA и имя API-сервера и предъявляет клиентский сертификат. API требует клиентский сертификат из локального CA и дополнительно разрешает только URI SAN `spiffe://mtusi2026lab7/load-balancer`. API listener не опубликован на host; internal health-check также использует TLS/mTLS. + +Пароли записываются как Argon2id (`m=19456 KiB`, `t=2`, `p=1`); параметры hash проверяются до верификации и ограничены верхним пределом 64 MiB. Одновременно выполняется не более четырёх password hash операций в каждом API процессе; при исчерпании слотов запрос ограниченно ждёт 100 ms и получает 503/Retry-After, а не неограниченно аллоцирует память. Проверка legacy scrypt поддержана для миграции пользователей lab6; при успешном входе старый hash обновляется. Токены сессий продолжают храниться в PostgreSQL как SHA-256 hashes. Ограничение входа — пять попыток на username и шестьдесят на source IP за минуту; HAProxy заменяет `X-Real-IP`, поэтому значение от внешнего клиента не доверяется. Состояние общее для API-реплик, ответ блокировки — 429 с Retry-After. + +OpenBao использует TLS, file storage в persistent volume и KV v2. Bootstrap создаёт отдельные AppRole и policy path для API, worker, publisher, migrator и reconciler. Приложения получают только собственные static KV values; операторский root token и unseal keys не передаются контейнерам. SecretID ограничен 24 часами, token TTL — 15 минут (max 1 час). Сертификат CA и private keys создаются скриптом на локальном стенде. + +Фактические адреса приложения заданы в `compose.yaml`; `.env.example` — шаблон параметров локального запуска. Из переменных примера Compose использует порт ingress, число/лимиты API, session TTL, worker concurrency и telemetry sampling. Runtime secrets читаются через AppRole; в контейнерах нет root token. Python-сервисы работают как непривилегированный `app`, с `read_only`, tmpfs, `cap_drop: ALL` и `no-new-privileges`. Одноразовый `openbao-data-init` запускается от root только для установки владельца persistent volume и имеет единственную capability `CHOWN`; сам OpenBao drops all, оставляя `SETUID`/`SETGID`, нужные upstream entrypoint. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Структура и контракт | `make check` | PASS: OpenAPI structure, local refs, JSON и Python syntax | `scripts/check_contract.py` | +| Схемы API | `make validate PYTHON=.venv/bin/python` | PASS: OpenAPI 3.1 и JSON Schema | `scripts/validate_openapi.py` | +| Криптографические примитивы | `make test-primitives PYTHON=.venv/bin/python` | PASS: Argon2id, legacy scrypt verify/upgrade, URI SAN allowlist | `tests/test_security_primitives.py` | +| Compose topology | `docker compose --env-file .env.example config --quiet` после генерации локальных секретов/PKI | PASS: Compose конфигурация разбирается | `compose.yaml` | +| Локальная PKI | `openssl verify -CAfile secrets/pki/ca.crt ...` | PASS: ingress, API и LB client chain; client URI SAN содержит разрешённую identity | `scripts/bootstrap-pki.py` | +| API smoke over HTTPS | `make test BASE_URL=https://localhost:8443 CA_FILE=...` | PASS на работающем Compose стенде | `tests/smoke.py` | +| Внешняя проверка TLS trust | `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: доверенный TLS проходит, пустой trust store отвергается | `tests/security.py` | +| Внутреннее mTLS | `make test-internal-mtls` | PASS: без cert, foreign CA и неверный URI SAN отклонены; разрешённая LB identity принята | `tests/internal_mtls.py`, `src/app.py`, `haproxy.cfg` | +| Login rate limit | `make test-login-limit BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: шестая неуспешная попытка получает 429/Retry-After | `tests/login_rate_limit.py`, `migrations/006_login_rate_limits.sql` | +| Внутренние TLS-соединения | Работающий Compose smoke, S3 attachment processing, worker broker consumer и проверки datasource из Grafana | PASS: PostgreSQL TLS, S3 HTTPS/mTLS, RabbitMQ AMQPS/mTLS, OTLP HTTPS/mTLS; из Grafana проверена CA-валидация Prometheus/Loki/Tempo | `compose.yaml`, `infra/postgres/`, `infra/otel/`, `src/object_store.py`, `src/jobs.py` | +| OpenBao AppRole | `make test-openbao-acl PYTHON=.venv/bin/python` | PASS: пять сервисов читают собственные KV paths, cross-service read получает 403; token TTL ограничен | `tests/openbao_acl.py`, `scripts/bootstrap-openbao.py`, `infra/openbao/` | +| Worker least privilege | `make test-worker-isolation` | PASS: DB identity `messenger_worker`; чтение `users`, S3 List и чтение originals/variants запрещены; нужные staging/originals/variants операции разрешены; RabbitMQ publish отклонён с 403 | `tests/worker_isolation.py`, `evidence/worker-isolation.json` | +| Track 1: компрометация и recovery | `make test-worker-rotation ROOT_TOKEN_FILE=...` | PASS: старая AppRole SecretID отозвана и отвергнута; новая identity аутентифицируется; задача `queued → ready`, SHA-256 original и message link сохранены | `tests/worker_identity_rotation.py`, `evidence/worker-rotation.json` | +| Ротация LB leaf-сертификата | `make test-leaf-rotation` | PASS: старый сертификат отвергнут, новый принят, HTTPS ingress healthy, Docker image IDs не изменились | `tests/leaf_certificate_rotation.py`, `evidence/leaf-rotation.json` | +| Redaction telemetry | `make test-telemetry-redaction BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: 6 синтетических значений отсутствуют в 11 фактических Loki entries и 1 Tempo trace | `tests/telemetry_redaction.py`, `evidence/telemetry-redaction.json` | +| Сохранность при restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: подтверждённое сообщение, attachment content и существующая сессия сохранены | `tests/resilience.py`, `evidence/persistence.json` | +| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: две реплики до отказа; 108 успешных записей, 0 ошибок; первое восстановление за 3.463 s | `tests/resilience.py`, `evidence/failover.json` | +| Argon2 resource budget | `make test-argon-budget` на API с лимитом 1 CPU/512 MiB | 20 verify операций: p50 23.007 ms, p95 28.288 ms, max 32.157 ms; из 12 одновременных запросов 6 выполнились и 6 получили capacity rejection, peak 4 активных hash операции, max RSS 162220 KiB (Linux ru_maxrss) | `tests/argon2_budget.py`, `evidence/argon2-budget.json` | +| Python dependency scan | `.venv/bin/pip-audit -r requirements.txt` | PASS: 39 runtime packages, 0 известных уязвимостей в момент сканирования | `requirements.txt`, `evidence/dependency-audit.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Трек 1 — последствия компрометации worker + +**Гипотеза.** Получив worker identity, атакующий может читать/писать staging bucket, писать originals/variants и потреблять process queue. Он не может читать пользовательские записи, перечислять buckets или читать originals/variants, публиковать свои задачи либо читать чужие OpenBao paths. После отзыва SecretID и его замены ожидающая задача должна завершиться без потери вложения или связи с сообщением. + +**Методика.** На работающем Compose стенде выполнены `make test-worker-isolation` и `make test-worker-rotation ROOT_TOKEN_FILE=`. Первый сценарий проверяет реальные PostgreSQL grants, SeaweedFS `AccessDenied`, staging read/write, запись originals/variants, очистку временных объектов и broker ACL `403`. Второй останавливает worker, создаёт attachment и message link, выдаёт новую AppRole SecretID, отзывает прежнюю, проверяет отказ старого credential, пересоздаёт только worker и ждёт окончания обработки через публичный API. Root token и SecretID в вывод не включаются. + +**Результат и сравнение.** В исходной общей схеме worker наследовал права общих application credentials; после разделения его DB role и S3/RabbitMQ identity имеют отдельные allowlists. В текущем измерении worker прочитал attachments, но получил отказ на `users`; S3 write для originals/variants и staging read/write прошли, а List и чтение originals/variants завершились `AccessDenied`; RabbitMQ отклонил публикацию с 403. Прежняя SecretID не прошла AppRole login, новая identity получила только собственный KV path. Attachment перешёл из `queued` в `ready`; SHA-256 исходных байтов и ссылка на attachment в сообщении сохранились. Машинные результаты сохранены в `evidence/worker-isolation.json` и `evidence/worker-rotation.json`. + +**Границы эксперимента.** Проверялись заданные операции одной worker identity и один queued recovery сценарий на локальном Compose стенде; это не нагрузочный тест и не модель компрометации Docker host. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверены сохранность при restart, отказ одной API-реплики и восстановление записи; результаты приведены в таблице и JSON evidence. Отдельно измерялась стоимость Argon2 verify в ограниченном API container. Повторяемый профиль для учебного стенда: + +```sh +make load PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 \ + CA_FILE="$PWD/secrets/pki/ca.crt" +make test-persistence ACTION_SCRIPT=scripts/restart.sh \ + BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-failover ACTION_SCRIPT=scripts/stop-one.sh \ + BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +``` | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Нет клиентского сертификата / foreign CA / неверный URI SAN | API TLS listener отвергает соединение; LB identity проходит | Runtime тест: все три недоверенных запроса отклонены, LB identity принята | TLS-handshake | +| Недоверенный CA внешнего endpoint | Проверка сертификата завершается TLS verification error | HTTPS smoke и проверка trust store выполнены | TLS-handshake | +| Пять неуспешных login попыток в пределах минуты | Следующая попытка получает 429 и Retry-After | Runtime тест: шестая попытка отклонена с 429/Retry-After | Окно лимита — 60 секунд | +| Отзыв worker SecretID при ожидающей задаче | Старое credential отклоняется, новая identity продолжает обработку | Runtime тест: old login denied; attachment `queued → ready`, содержимое и message link сохранены | Проверен один recovery цикл | +| Restart API и stores | Сохранение подтверждённых данных, attachment content и существующей сессии | Runtime test: все данные доступны после restart | Проверка готовности после возврата сервисов | +| Отказ одной API-реплики | Успешные записи через оставшуюся реплику, без дублей и потери подтверждённых данных | 108 успешных записей, 0 ошибок; 2 реплики до отказа | Первое восстановление за 3.463 s | -## Ограничения и следующие шаги +`tests/load.py` — closed-loop генератор; его успешный HTTP throughput не является точным числом DB commits и не компенсирует coordinated omission. Измерения необходимо выполнять на одном и том же составе стенда с зафиксированными concurrency, длительностью, прогревом, аппаратными ресурсами и размером данных; сохранять вывод в `evidence/` без credentials и токенов. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Дополнительные измерения + +Ротация LB leaf-сертификата проверена без пересборки образов: прежний сертификат отклоняется, новый проходит ingress health check. Redaction проверен на фактических записях Loki и трассе Tempo после генерации синтетических credentials и сообщения. Фактические протоколы и identities каждой связи зафиксированы в [матрице доверия](contracts/trust-matrix.md); остальные проверки и команды воспроизведения приведены в [tests/README.md](tests/README.md). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..a75426d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,871 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + - postgres_tls_cert + - ca_crt + command: + - postgres + - -c + - ssl=on + - -c + - ssl_cert_file=/tls/server.crt + - -c + - ssl_key_file=/tls/server.key + - -c + - ssl_min_protocol_version=TLSv1.2 + - -c + - hba_file=/etc/postgresql/pg_hba.conf + volumes: + - postgres_data:/var/lib/postgresql/data + - postgres_tls:/tls + - ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + depends_on: + postgres-tls-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + secrets: + - source: rabbitmq_config + target: /etc/rabbitmq/conf.d/10-defaults.conf + - ca_crt + - rabbitmq_tls_cert + - rabbitmq_tls_key + - rabbitmq_definitions + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + restart: unless-stopped + + s3-core: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - server + - -dir=/data + - -filer=true + - -s3=false + - -volume.max=21 + volumes: + - seaweed_data:/data + healthcheck: + test: + - CMD-SHELL + - "curl --fail --silent http://127.0.0.1:8888/ >/dev/null" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + networks: + - seaweed_internal + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - s3 + - -filer=s3-core:8888 + - -config=/run/secrets/s3_config + - -iam.config=/run/secrets/s3_iam_config + - -port=0 + - -port.https=8334 + - -cert.file=/run/secrets/s3_tls_cert + - -key.file=/run/secrets/s3_tls_key + - -cacert.file=/run/secrets/ca_crt + - -tlsVerifyClientCert=true + - -port.grpc=0 + - -port.iceberg=0 + - -port.lance=0 + - -concurrentFileUploadLimit=10 + - -concurrentUploadLimitMB=100 + - -readerCacheSizeMB=64 + - -autoCreateBucket=false + - -ip.bind=0.0.0.0 + secrets: + - s3_config + - s3_iam_config + - s3_tls_cert + - s3_tls_key + - ca_crt + - s3_healthcheck_client_cert + - s3_healthcheck_client_key + depends_on: + s3-core: + condition: service_healthy + networks: + - default + - seaweed_internal + healthcheck: + test: + - CMD-SHELL + - "code=$$(curl --cacert /run/secrets/ca_crt --cert /run/secrets/s3_healthcheck_client_cert --key /run/secrets/s3_healthcheck_client_key -sS -o /dev/null -w '%{http_code}' https://s3:8334/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3-bootstrap: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/bootstrap_s3.py"] + environment: + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_bootstrap_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_bootstrap_client_key + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - ca_crt + - s3_bootstrap_client_cert + - s3_bootstrap_client_key + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + postgres-tls-init: + image: alpine:3.21.3 + user: "0:0" + command: + - sh + - -c + - "cp /run/secrets/postgres_tls_cert /tls/server.crt && cp /run/secrets/postgres_tls_key /tls/server.key && chmod 0644 /tls/server.crt && chmod 0600 /tls/server.key && chown 70:70 /tls/server.crt /tls/server.key" + secrets: + - postgres_tls_cert + - postgres_tls_key + volumes: + - postgres_tls:/tls + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + cap_drop: + - ALL + cap_add: + - CHOWN + security_opt: + - no-new-privileges:true + restart: "no" + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/migrate.py"] + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_crt + BAO_ROLE_ID_FILE: /run/secrets/migrate_role_id + BAO_SECRET_ID_FILE: /run/secrets/migrate_secret_id + BAO_SECRET_PATH: messenger/migrate + WORKER_DB_PASSWORD_FILE: /run/secrets/worker_db_password + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_migrate_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_migrate_client_key + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + DB_POOL_MAX_SIZE: "2" + secrets: + - ca_crt + - migrate_role_id + - migrate_secret_id + - worker_db_password + - s3_migrate_client_cert + - s3_migrate_client_key + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=32m + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + TLS_CERT_FILE: /run/secrets/api_tls_cert + TLS_KEY_FILE: /run/secrets/api_tls_key + TLS_CA_FILE: /run/secrets/ca_crt + TLS_CLIENT_CERT_FILE: /run/secrets/lb_tls_cert + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_crt + BAO_ROLE_ID_FILE: /run/secrets/api_role_id + BAO_SECRET_ID_FILE: /run/secrets/api_secret_id + BAO_SECRET_PATH: messenger/api + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_api_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_api_client_key + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + ARGON2_MEMORY_KIB: "19456" + ARGON2_ITERATIONS: "2" + ARGON2_PARALLELISM: "1" + PASSWORD_HASH_CONCURRENCY: "4" + PASSWORD_HASH_WAIT_SECONDS: "0.1" + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10} + UPLOAD_TIMEOUT_SECONDS: "60" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX_SIZE: "8" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + DB_LOCK_TIMEOUT_MS: "2000" + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_api_client_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_api_client_key + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - ca_crt + - api_tls_cert + - api_tls_key + - lb_tls_cert + - lb_tls_key + - wrong_client_cert + - wrong_client_key + - foreign_client_cert + - foreign_client_key + - api_role_id + - api_secret_id + - otel_api_client_cert + - otel_api_client_key + - s3_api_client_cert + - s3_api_client_key + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + depends_on: + migrate: + condition: service_completed_successfully + openbao: + condition: service_started + otel-collector: + condition: service_started + healthcheck: + test: + - CMD + - python3 + - -c + - "import ssl,urllib.request; c=ssl.create_default_context(cafile='/run/secrets/ca_crt'); c.load_cert_chain('/run/secrets/lb_tls_cert','/run/secrets/lb_tls_key'); urllib.request.urlopen('https://127.0.0.1:8080/health/ready', context=c, timeout=2).read()" + interval: 3s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEMORY_LIMIT:-512M}" + stop_grace_period: 20s + restart: unless-stopped + + publisher: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.publisher"] + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_crt + BAO_ROLE_ID_FILE: /run/secrets/publisher_role_id + BAO_SECRET_ID_FILE: /run/secrets/publisher_secret_id + BAO_SECRET_PATH: messenger/publisher + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_publisher_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_publisher_client_key + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + DB_POOL_MAX_SIZE: "2" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-publisher + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_publisher_client_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_publisher_client_key + BROKER_CA_FILE: /run/secrets/ca_crt + BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_publisher_client_cert + BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_publisher_client_key + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - ca_crt + - publisher_role_id + - publisher_secret_id + - otel_publisher_client_cert + - otel_publisher_client_key + - rabbitmq_publisher_client_cert + - rabbitmq_publisher_client_key + - s3_publisher_client_cert + - s3_publisher_client_key + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=32m + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + depends_on: + migrate: + condition: service_completed_successfully + openbao: + condition: service_started + rabbitmq: + condition: service_healthy + otel-collector: + condition: service_started + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "-m", "src.worker"] + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_crt + BAO_ROLE_ID_FILE: /run/secrets/worker_role_id + BAO_SECRET_ID_FILE: /run/secrets/worker_secret_id + BAO_SECRET_PATH: messenger/worker + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_worker_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_worker_client_key + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + MAX_IMAGE_PIXELS: "40000000" + DB_POOL_MAX_SIZE: "2" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + APP_ENV: lab + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_worker_client_cert + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_worker_client_key + BROKER_CA_FILE: /run/secrets/ca_crt + BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_worker_client_cert + BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_worker_client_key + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + secrets: + - ca_crt + - worker_role_id + - worker_secret_id + - otel_worker_client_cert + - otel_worker_client_key + - rabbitmq_worker_client_cert + - rabbitmq_worker_client_key + - s3_worker_client_cert + - s3_worker_client_key + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + depends_on: + migrate: + condition: service_completed_successfully + openbao: + condition: service_started + rabbitmq: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + otel-collector: + condition: service_started + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + reconciler: + build: + context: . + dockerfile: Dockerfile + command: ["sh", "scripts/reconcile-loop.sh"] + environment: + BAO_ADDR: https://openbao:8200 + BAO_CACERT: /run/secrets/ca_crt + BAO_ROLE_ID_FILE: /run/secrets/reconciler_role_id + BAO_SECRET_ID_FILE: /run/secrets/reconciler_secret_id + BAO_SECRET_PATH: messenger/reconciler + S3_ENDPOINT: https://s3:8334 + S3_CA_FILE: /run/secrets/ca_crt + S3_CLIENT_CERT_FILE: /run/secrets/s3_reconciler_client_cert + S3_CLIENT_KEY_FILE: /run/secrets/s3_reconciler_client_key + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_STAGING_BUCKET: messenger-staging + S3_VARIANTS_BUCKET: messenger-variants + DB_POOL_MAX_SIZE: "2" + RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600} + secrets: + - ca_crt + - reconciler_role_id + - reconciler_secret_id + - s3_reconciler_client_cert + - s3_reconciler_client_key + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=32m + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + depends_on: + migrate: + condition: service_completed_successfully + openbao: + condition: service_started + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.163.2 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + secrets: + - ca_crt + - otel_collector_tls_cert + - otel_collector_tls_key + - tempo_tls_cert + - loki_tls_cert + depends_on: + - tempo + - loki + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + prometheus: + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --web.config.file=/etc/prometheus/web.yml + secrets: + - prometheus_tls_cert + - prometheus_tls_key + - ca_crt + - collector_prometheus_client_cert + - collector_prometheus_client_key + volumes: + - ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/otel/prometheus-web.yml:/etc/prometheus/web.yml:ro + - ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + depends_on: + - otel-collector + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2 + command: ["-config.file=/etc/loki/loki.yml"] + secrets: + - loki_tls_cert + - loki_tls_key + volumes: + - ./infra/otel/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.1 + command: ["-config.file=/etc/tempo/tempo.yml"] + secrets: + - tempo_tls_cert + - tempo_tls_key + volumes: + - ./infra/otel/tempo.yml:/etc/tempo/tempo.yml:ro + - tempo_data:/var/tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + grafana: + image: grafana/grafana:11.4.0 + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + SSL_CERT_FILE: /run/secrets/ca_crt + secrets: + - grafana_admin_password + - ca_crt + volumes: + - grafana_data:/var/lib/grafana + - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + - tempo + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.7-alpine + ports: + - "127.0.0.1:${APP_PORT:-8443}:8443" + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + secrets: + - ingress_pem + - ca_crt + - lb_client_pem + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: unless-stopped + + openbao: + image: quay.io/openbao/openbao:2.1.1 + command: ["server", "-config=/bao/config.hcl"] + ports: + - "127.0.0.1:8200:8200" + volumes: + - openbao_data:/openbao/data + - ./infra/openbao/config.hcl:/bao/config.hcl:ro + secrets: + - openbao_tls_cert + - openbao_tls_key + - ca_crt + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + depends_on: + openbao-data-init: + condition: service_completed_successfully + security_opt: + - no-new-privileges:true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: unless-stopped + + openbao-data-init: + image: alpine:3.21.3 + user: "0:0" + command: ["sh", "-c", "mkdir -p /openbao/data && chown 100:1000 /openbao/data"] + volumes: + - openbao_data:/openbao/data + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + cap_drop: + - ALL + cap_add: + - CHOWN + security_opt: + - no-new-privileges:true + restart: "no" + +secrets: + postgres_password: + file: ./secrets/postgres_password + worker_db_password: + file: ./secrets/worker_db_password + postgres_tls_cert: + file: ./secrets/pki/postgres.crt + postgres_tls_key: + file: ./secrets/pki/postgres.key + s3_bootstrap_client_cert: + file: ./secrets/pki/s3-bootstrap-client.crt + s3_bootstrap_client_key: + file: ./secrets/pki/s3-bootstrap-client.key + s3_healthcheck_client_cert: + file: ./secrets/pki/s3-healthcheck-client.crt + s3_healthcheck_client_key: + file: ./secrets/pki/s3-healthcheck-client.key + s3_api_client_cert: + file: ./secrets/pki/s3-api-client.crt + s3_api_client_key: + file: ./secrets/pki/s3-api-client.key + s3_worker_client_cert: + file: ./secrets/pki/s3-worker-client.crt + s3_worker_client_key: + file: ./secrets/pki/s3-worker-client.key + s3_publisher_client_cert: + file: ./secrets/pki/s3-publisher-client.crt + s3_publisher_client_key: + file: ./secrets/pki/s3-publisher-client.key + s3_migrate_client_cert: + file: ./secrets/pki/s3-migrate-client.crt + s3_migrate_client_key: + file: ./secrets/pki/s3-migrate-client.key + s3_reconciler_client_cert: + file: ./secrets/pki/s3-reconciler-client.crt + s3_reconciler_client_key: + file: ./secrets/pki/s3-reconciler-client.key + database_url: + file: ./secrets/database_url + s3_config: + file: ./secrets/s3.json + s3_iam_config: + file: ./secrets/s3_iam.json + s3_tls_cert: + file: ./secrets/pki/s3.crt + s3_tls_key: + file: ./secrets/pki/s3.key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + rabbitmq_password: + file: ./secrets/rabbitmq_password + rabbitmq_config: + file: ./secrets/rabbitmq.conf + rabbitmq_definitions: + file: ./secrets/rabbitmq_definitions.json + broker_url: + file: ./secrets/broker_url + broker_worker_url: + file: ./secrets/broker_worker_url + broker_publisher_url: + file: ./secrets/broker_publisher_url + grafana_admin_password: + file: ./secrets/grafana_admin_password + ca_crt: + file: ./secrets/pki/ca.crt + api_tls_cert: + file: ./secrets/pki/api.crt + api_tls_key: + file: ./secrets/pki/api.key + lb_tls_cert: + file: ./secrets/pki/lb-client.crt + lb_tls_key: + file: ./secrets/pki/lb-client.key + lb_client_pem: + file: ./secrets/pki/lb-client.pem + ingress_pem: + file: ./secrets/pki/ingress.pem + openbao_tls_cert: + file: ./secrets/pki/openbao.crt + openbao_tls_key: + file: ./secrets/pki/openbao.key + rabbitmq_tls_cert: + file: ./secrets/pki/rabbitmq.crt + rabbitmq_tls_key: + file: ./secrets/pki/rabbitmq.key + otel_collector_tls_cert: + file: ./secrets/pki/otel-collector.crt + otel_collector_tls_key: + file: ./secrets/pki/otel-collector.key + tempo_tls_cert: + file: ./secrets/pki/tempo.crt + tempo_tls_key: + file: ./secrets/pki/tempo.key + loki_tls_cert: + file: ./secrets/pki/loki.crt + loki_tls_key: + file: ./secrets/pki/loki.key + prometheus_tls_cert: + file: ./secrets/pki/prometheus.crt + prometheus_tls_key: + file: ./secrets/pki/prometheus.key + collector_prometheus_client_cert: + file: ./secrets/pki/collector-prometheus-client.crt + collector_prometheus_client_key: + file: ./secrets/pki/collector-prometheus-client.key + otel_api_client_cert: + file: ./secrets/pki/otel-api-client.crt + otel_api_client_key: + file: ./secrets/pki/otel-api-client.key + otel_worker_client_cert: + file: ./secrets/pki/otel-worker-client.crt + otel_worker_client_key: + file: ./secrets/pki/otel-worker-client.key + otel_publisher_client_cert: + file: ./secrets/pki/otel-publisher-client.crt + otel_publisher_client_key: + file: ./secrets/pki/otel-publisher-client.key + rabbitmq_api_client_cert: + file: ./secrets/pki/rabbitmq-api-client.crt + rabbitmq_api_client_key: + file: ./secrets/pki/rabbitmq-api-client.key + rabbitmq_worker_client_cert: + file: ./secrets/pki/rabbitmq-worker-client.crt + rabbitmq_worker_client_key: + file: ./secrets/pki/rabbitmq-worker-client.key + rabbitmq_publisher_client_cert: + file: ./secrets/pki/rabbitmq-publisher-client.crt + rabbitmq_publisher_client_key: + file: ./secrets/pki/rabbitmq-publisher-client.key + wrong_client_cert: + file: ./secrets/pki/wrong-client.crt + wrong_client_key: + file: ./secrets/pki/wrong-client.key + foreign_client_cert: + file: ./secrets/pki/foreign-client.crt + foreign_client_key: + file: ./secrets/pki/foreign-client.key + api_role_id: + file: ./secrets/openbao/api_role_id + api_secret_id: + file: ./secrets/openbao/api_secret_id + worker_role_id: + file: ./secrets/openbao/worker_role_id + worker_secret_id: + file: ./secrets/openbao/worker_secret_id + publisher_role_id: + file: ./secrets/openbao/publisher_role_id + publisher_secret_id: + file: ./secrets/openbao/publisher_secret_id + migrate_role_id: + file: ./secrets/openbao/migrate_role_id + migrate_secret_id: + file: ./secrets/openbao/migrate_secret_id + reconciler_role_id: + file: ./secrets/openbao/reconciler_role_id + reconciler_secret_id: + file: ./secrets/openbao/reconciler_secret_id + +volumes: + postgres_data: + postgres_tls: + seaweed_data: + rabbitmq_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + openbao_data: + +networks: + seaweed_internal: + internal: true diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..e5b7ec2 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,35 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия — lab7 -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +Для каждой связи указаны используемые endpoint, protocol, identity и фактический способ проверки. Состояние относится к реализации в `compose.yaml`. -| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | +| Связь | Фактический endpoint / protocol / identity | Проверка, срок действия | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | `localhost:8443`, HTTPS; CA `secrets/pki/ca.crt`; ingress SAN `localhost`, `127.0.0.1`; внешний client certificate не требуется | Python HTTPS client проверяет CA и hostname; пользовательская идентичность — Bearer session; пять попыток на username и шестьдесят на source IP за минуту, затем 429/Retry-After | `haproxy.cfg`, `scripts/bootstrap-pki.py`, `tests/security.py`, `tests/login_rate_limit.py` | +| LB → API (все реплики) | `api:8080`, TLS 1.2+, mTLS; CA `ca.crt`; HAProxy cert URI SAN `spiffe://mtusi2026lab7/load-balancer`; API SAN `DNS:api` | HAProxy проверяет CA и DNS hostname; API проверяет клиентскую CA chain и точный URI SAN; leaf lifetime 397 дней | `haproxy.cfg`, `src/app.py`, `src/security.py`, `tests/internal_mtls.py`; `make test-internal-mtls` | +| API/worker → PostgreSQL | `postgres:5432`, PostgreSQL TLS over Compose network; CA verification (`sslmode=verify-full`); separate API and `messenger_worker` DB roles | Server certificate and hostname verified; worker role limited to attachment/outbox operations and columns | `compose.yaml`, `infra/postgres/pg_hba.conf`, `src/store.py`, `tests/worker_isolation.py` | +| API/worker/publisher/reconciler → S3 | `s3:8334`, HTTPS with mTLS; separate client certificates and access keys; CA verification | API reads originals/variants and lists originals as configured; worker reads/writes staging and writes originals/variants, but cannot list or read originals/variants; publisher/migrator have no object actions | `compose.yaml`, `scripts/prepare-secure-secrets.py`, `src/object_store.py`, `tests/worker_isolation.py` | +| Publisher/worker → broker | `rabbitmq:5671`, AMQPS with mTLS and CA/hostname verification; separate publisher and worker identities | Publisher can publish through the default exchange; worker consumes `attachments.process.v1` and is denied publishing; queue definitions are provisioned separately | `compose.yaml`, `src/jobs.py`, `scripts/prepare-secure-secrets.py`, `tests/worker_isolation.py` | +| API/worker/publisher → Collector | `otel-collector:4318`, OTLP/HTTP over HTTPS with mTLS; distinct client certificates and CA verification | Collector requires a client certificate from the lab CA; application uses TLS exporter settings | `compose.yaml`, `infra/otel/collector.yaml`, `src/telemetry.py` | +| Collector → telemetry backends | Prometheus scrape endpoint uses HTTPS/mTLS; Loki and Tempo use HTTPS with CA validation | Prometheus client certificate is checked by Collector; Collector validates backend CA | `infra/otel/collector.yaml`, `infra/otel/prometheus.yml`, `infra/otel/loki.yml`, `infra/otel/tempo.yml` | +| Grafana → data sources | Prometheus, Loki and Tempo HTTPS service endpoints | Grafana uses the local CA trust bundle; endpoints were queried from the Grafana container | `compose.yaml`, `infra/otel/grafana/provisioning/datasources/datasources.yml` | +| API/worker/agent → OpenBao | `https://openbao:8200`, CA verified; AppRole role_id + SecretID; KV v2 service-specific path | Per-service policy разрешает только `secret/data/messenger/` read; SecretID TTL 24h, service token TTL 15m (max 1h) | `infra/openbao/config.hcl`, `scripts/bootstrap-openbao.py`, `src/config.py` | +| Оператор → OpenBao/Grafana/S3 admin | OpenBao опубликован только на `127.0.0.1:8200` по TLS; Grafana также на loopback; S3 admin используется bootstrap job внутри Compose network | OpenBao root token и unseal shares остаются у оператора; application services их не получают | `compose.yaml`, `infra/openbao/README.md`, `scripts/bootstrap-secrets.py` | +| API → Redis/Valkey, если используется | N/A: session source of truth — PostgreSQL; Redis/Valkey не выбран | Не применяется | `src/store.py`, `compose.yaml` | +| Replication/кластерные каналы, если добавлены | N/A: replicated database, broker cluster и multi-host transport не добавлены | Не применяется | `compose.yaml` | +| LB/API health checks | LB health request использует mTLS backend connection; API self-check использует loopback HTTPS и разрешённый LB certificate | API listener не опубликован на host; health endpoint не обходит mTLS | `haproxy.cfg`, `compose.yaml`, `src/app.py` | -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +## Отрицательные проверки -## Обязательные отрицательные проверки - -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. +1. `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` выполняет положительный TLS probe и проверку отказа пустого trust store. Hostname проверяется клиентским TLS context. +2. `make test-internal-mtls` проверяет API listener без client cert, с cert от foreign CA и с URI SAN другой службы; разрешённый LB cert должен пройти. +3. `make test-login-limit` проверяет 429 и положительный Retry-After после неудачных попыток. +4. `make test` проверяет ACL чат-участников, logout, сессии, сообщения и вложения; дополнительный `X-User-Id` не является источником identity после lab1. +5. `make test-openbao-acl` проверяет, что каждая AppRole читает только собственный KV path и получает 403 на остальные service paths. +6. `make test-worker-isolation` проверяет разрешённые и запрещённые операции worker в PostgreSQL, S3 и RabbitMQ. +7. `make test-worker-rotation ROOT_TOKEN_FILE=...` отзывает старый worker SecretID, проверяет отказ старой identity и завершает queued attachment с заменённой identity. ## Bootstrap и эксплуатация -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. +В Git хранятся только конфигурация и скрипты. CA private key, leaf keys, root token, unseal shares и AppRole SecretIDs создаются локально и хранятся в исключённом каталоге `secrets/`. Оператор отвечает за CA/unseal; OpenBao использует persistent file storage, TLS listener и AppRole auth. Root token не монтируется приложению. После restart оператор unseal-ит OpenBao при необходимости; приложение повторно аутентифицируется AppRole при старте. -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +Redis и кластерные каналы не используются. После перезапуска OpenBao operator выполняет unseal; приложение не использует plaintext fallback при недоступном/sealed OpenBao. diff --git a/evidence/argon2-budget.json b/evidence/argon2-budget.json new file mode 100644 index 0000000..31dcca6 --- /dev/null +++ b/evidence/argon2-budget.json @@ -0,0 +1 @@ +{"argon2_iterations": 2, "argon2_memory_kib": 19456, "argon2_parallelism": 1, "burst_capacity_rejections": 6, "burst_requests": 12, "burst_successes": 6, "hash_slots_per_process": 4, "hash_wait_seconds": 0.1, "max_rss_platform_units": 162220, "peak_active_hashes": 4, "platform": "linux", "sequential_samples": 20, "verify_latency_ms": {"max": 32.157, "p50": 23.007, "p95": 28.288}} diff --git a/evidence/dependency-audit.json b/evidence/dependency-audit.json new file mode 100644 index 0000000..43db51d --- /dev/null +++ b/evidence/dependency-audit.json @@ -0,0 +1 @@ +{"dependencies": [{"name": "psycopg", "version": "3.2.10", "vulns": []}, {"name": "psycopg-pool", "version": "3.3.3", "vulns": []}, {"name": "redis", "version": "5.2.1", "vulns": []}, {"name": "boto3", "version": "1.35.99", "vulns": []}, {"name": "pillow", "version": "13.0.0", "vulns": []}, {"name": "pika", "version": "1.3.2", "vulns": []}, {"name": "opentelemetry-api", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-sdk", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-http", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation-psycopg", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-botocore", "version": "0.66b2", "vulns": []}, {"name": "psutil", "version": "6.1.1", "vulns": []}, {"name": "argon2-cffi", "version": "23.1.0", "vulns": []}, {"name": "opentelemetry-exporter-http-transport", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-common", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-common", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-dbapi", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-proto", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-semantic-conventions", "version": "0.66b2", "vulns": []}, {"name": "psycopg-binary", "version": "3.2.10", "vulns": []}, {"name": "botocore", "version": "1.35.99", "vulns": []}, {"name": "googleapis-common-protos", "version": "1.76.0", "vulns": []}, {"name": "jmespath", "version": "1.1.0", "vulns": []}, {"name": "opentelemetry-propagator-aws-xray", "version": "1.0.2", "vulns": []}, {"name": "protobuf", "version": "7.36.2", "vulns": []}, {"name": "python-dateutil", "version": "2.9.0.post0", "vulns": []}, {"name": "requests", "version": "2.34.2", "vulns": []}, {"name": "charset-normalizer", "version": "3.5.2", "vulns": []}, {"name": "idna", "version": "3.20", "vulns": []}, {"name": "s3transfer", "version": "0.10.4", "vulns": []}, {"name": "urllib3", "version": "2.8.0", "vulns": []}, {"name": "wrapt", "version": "2.5.2", "vulns": []}, {"name": "certifi", "version": "2026.10.23", "vulns": []}, {"name": "six", "version": "1.17.0", "vulns": []}, {"name": "typing-extensions", "version": "4.16.0", "vulns": []}, {"name": "argon2-cffi-bindings", "version": "26.1.0", "vulns": []}, {"name": "cffi", "version": "2.1.1", "vulns": []}, {"name": "pycparser", "version": "3.12", "vulns": []}], "fixes": []} diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 0000000..48c1b8d --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,7 @@ +{ + "failed_attempts": 0, + "first_recovery_seconds": 3.463, + "observed_instances_before": 2, + "successful_writes": 108, + "result": "PASS" +} diff --git a/evidence/leaf-rotation.json b/evidence/leaf-rotation.json new file mode 100644 index 0000000..8cb1cdd --- /dev/null +++ b/evidence/leaf-rotation.json @@ -0,0 +1 @@ +{"https_ingress_healthy": true, "images_unchanged": true, "new_leaf_accepted": true, "new_leaf_sha256": "55e47bba381c871fa4c4dbdf34994526247be6429e37ae071199e73fa71d21ea", "old_leaf_rejected": true, "result": "PASS"} diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 0000000..f167ec3 --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,7 @@ +{ + "attachment_content_preserved": true, + "confirmed_message_preserved": true, + "existing_session_preserved": true, + "hook": "scripts/restart.sh", + "result": "PASS" +} diff --git a/evidence/pipeline-faults.json b/evidence/pipeline-faults.json new file mode 100644 index 0000000..09ce006 --- /dev/null +++ b/evidence/pipeline-faults.json @@ -0,0 +1,58 @@ +{ + "fixture": "tests/fixtures/sample.png", + "invariants": { + "ambiguous_confirm_republishes_stable_job": true, + "confirmed_work_eventually_ready": true, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + "duplicate_effects_create_no_second_variant_set": true, + "stale_attempt_writes_no_effects": true, + "stale_pipeline_cannot_publish_current_metadata": true + }, + "result": "PASS", + "scenarios": [ + { + "deliveries": 2, + "duplicate_delivery_writes": 0, + "effective_object_writes": 4, + "scenario": "duplicate_delivery_after_commit_before_ack" + }, + { + "attempts": 2, + "scenario": "crash_after_claim", + "terminal_status": "ready" + }, + { + "attempts": 2, + "scenario": "crash_after_original_put", + "terminal_status": "ready" + }, + { + "current_attempt": 2, + "scenario": "out_of_order_attempt_delivery", + "stale_attempt_extra_writes": 0 + }, + { + "current_pipeline_version": 2, + "published_variant_versions": [ + "v2" + ], + "scenario": "stale_pipeline_after_newer_version" + }, + { + "outbox_recovered_after_restart": true, + "published_after_restart": 1, + "scenario": "publisher_crash_before_publish" + }, + { + "deliveries": 2, + "same_message_id": true, + "scenario": "publish_confirm_lost_after_broker_accept" + }, + { + "configured_timeout_seconds": 1, + "observed_seconds": 1.007, + "scenario": "transform_timeout_terminated" + } + ], + "track": "2" +} diff --git a/evidence/telemetry-redaction.json b/evidence/telemetry-redaction.json new file mode 100644 index 0000000..3ccef74 --- /dev/null +++ b/evidence/telemetry-redaction.json @@ -0,0 +1 @@ +{"api_traces_scanned": 1, "leaks": 0, "log_entries_scanned": 11, "result": "PASS", "synthetic_values_checked": 6} diff --git a/evidence/worker-isolation.json b/evidence/worker-isolation.json new file mode 100644 index 0000000..a6f99ee --- /dev/null +++ b/evidence/worker-isolation.json @@ -0,0 +1 @@ +{"broker": {"broker_identity_connected": true, "publish_denied": true}, "database": {"attachments_read": true, "role": "messenger_worker", "users_read_denied": true}, "result": "PASS", "s3": {"messenger-attachments_list_denied": true, "messenger-attachments_read_denied": true, "messenger-attachments_write": true, "messenger-staging_list_denied": true, "messenger-staging_read": true, "messenger-staging_write": true, "messenger-variants_list_denied": true, "messenger-variants_read_denied": true, "messenger-variants_write": true, "temporary_objects_removed": true}} diff --git a/evidence/worker-rotation.json b/evidence/worker-rotation.json new file mode 100644 index 0000000..9355406 --- /dev/null +++ b/evidence/worker-rotation.json @@ -0,0 +1 @@ +{"attachment_state": ["queued", "ready"], "message_attachment_link_preserved": true, "old_worker_secret_id_revoked": true, "original_sha256_preserved": true, "replacement_worker_secret_id_authenticated": true, "result": "PASS"} diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 0000000..b6539b7 --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + option http-keep-alive + timeout connect 2s + timeout client 15s + timeout server 15s + timeout http-request 5s + timeout check 1s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend messenger + bind :8443 ssl crt /run/secrets/ingress_pem + http-request set-header X-Real-IP %[src] + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk + http-check connect ssl sni api + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + option redispatch + default-server inter 1s fall 2 rise 2 + server-template api 1-10 api:8080 check ssl verify required ca-file /run/secrets/ca_crt crt /run/secrets/lb_client_pem verifyhost api sni str(api) resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/infra/README.md b/infra/README.md index 5a39e86..877197c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,6 +1,6 @@ # Инфраструктура студента -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Конфигурации выбранных сервисов находятся здесь; основная топология — в корневом `compose.yaml`. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. Текущие OpenBao policies/bootstrap описаны в `openbao/`; OTel provisioning перенесён из lab6 в `otel/`. В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..6a68f7c 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -1,7 +1,14 @@ -# OpenBao — реализовать конфигурацию и политики +# OpenBao service configuration and bootstrap -Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key. +`config.hcl` configures a TLS listener and persistent file storage; Compose runs OpenBao outside dev mode. `bootstrap-openbao.py` provisions KV v2 secrets and per-process AppRoles after operator init/unseal. -Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. +The API, worker, publisher, migrator and reconciler use separate AppRole credentials and policies. Each policy can read only its own `secret/data/messenger/` KV v2 path. Service tokens have a 15-minute TTL and a one-hour maximum TTL; AppRole SecretIDs expire after 24 hours. Application code never receives the operator root token. -Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. +Operator workflow: + +1. Generate local keys and bootstrap secrets with the scripts documented in the root README. +2. Start the `openbao` service, run `bao operator init`, and unseal it using the operator-held shares. Keep the initial root token outside the repository. +3. Run `python3 scripts/bootstrap-openbao.py --root-token-file `. It writes non-root AppRole IDs/SecretIDs under ignored `secrets/openbao/`; values are not printed. +4. Start application services. On a normal restart, the file backend retains OpenBao state and the operator unseals it if required. If a service SecretID expires, provision a replacement and restart that service. + +The script provisions distinct paths and identities; secret values originate from local bootstrap files and are not committed. Root token and unseal shares must never be mounted into application containers. diff --git a/infra/openbao/api-policy.hcl b/infra/openbao/api-policy.hcl new file mode 100644 index 0000000..190c529 --- /dev/null +++ b/infra/openbao/api-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/messenger/api" { + capabilities = ["read"] +} diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl new file mode 100644 index 0000000..b0225dc --- /dev/null +++ b/infra/openbao/config.hcl @@ -0,0 +1,15 @@ +ui = false +api_addr = "https://openbao:8200" +cluster_addr = "https://openbao:8201" + +storage "file" { + path = "/openbao/data" +} + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + tls_cert_file = "/run/secrets/openbao_tls_cert" + tls_key_file = "/run/secrets/openbao_tls_key" + tls_min_version = "tls12" +} diff --git a/infra/openbao/worker-policy.hcl b/infra/openbao/worker-policy.hcl new file mode 100644 index 0000000..6c67e26 --- /dev/null +++ b/infra/openbao/worker-policy.hcl @@ -0,0 +1,3 @@ +path "secret/data/messenger/worker" { + capabilities = ["read"] +} diff --git a/infra/otel/alerts.yml b/infra/otel/alerts.yml new file mode 100644 index 0000000..50e245a --- /dev/null +++ b/infra/otel/alerts.yml @@ -0,0 +1,42 @@ +groups: + - name: messenger-slo + interval: 10s + rules: + - record: messenger:api_5xx_ratio:rate5m + expr: | + (sum(rate(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0)) + / + clamp_min(sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])), 1e-9) + - record: messenger:api_availability:ratio7d + expr: | + 1 - + (sum(increase(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[7d])) or vector(0)) + / + clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[7d])), 1) + - alert: MessengerApiErrorBudgetBurn + expr: messenger:api_5xx_ratio:rate5m > 0.01 + for: 2m + labels: + severity: warning + annotations: + summary: API error budget burn exceeds the 1% short-window budget + description: The rolling 5-minute business API 5xx ratio is above 1%. + runbook: Check API traces for the failing route, then inspect correlated logs and DB/S3 spans. + - alert: MessengerQueueLag + expr: max(messenger_queue_oldest_age_seconds) > 60 and max(messenger_queue_depth) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Oldest attachment job has waited for more than 60 seconds + description: Inspect worker availability, RabbitMQ health, and the worker trace for the oldest job. + runbook: Check queue depth and age, worker consuming logs, broker, then recover the worker and confirm resolved state. + - alert: MessengerDatabasePoolSaturation + expr: max(messenger_db_pool_requests_waiting) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: Requests are waiting for a database connection + description: Compare DB waiters with request traces and PostgreSQL health. + runbook: Inspect DB spans and pool use; verify database health and connection limits. diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..9466be3 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,80 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: /run/secrets/otel_collector_tls_cert + key_file: /run/secrets/otel_collector_tls_key + client_ca_file: /run/secrets/ca_crt + http: + endpoint: 0.0.0.0:4318 + tls: + cert_file: /run/secrets/otel_collector_tls_cert + key_file: /run/secrets/otel_collector_tls_key + client_ca_file: /run/secrets/ca_crt + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 384 + spike_limit_mib: 96 + batch: + timeout: 1s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + tls: + cert_file: /run/secrets/otel_collector_tls_cert + key_file: /run/secrets/otel_collector_tls_key + client_ca_file: /run/secrets/ca_crt + send_timestamps: true + resource_to_telemetry_conversion: + enabled: true + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + ca_file: /run/secrets/ca_crt + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: https://loki:3100/otlp + tls: + ca_file: /run/secrets/ca_crt + sending_queue: + enabled: true + queue_size: 256 + num_consumers: 2 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + telemetry: + logs: + level: warn + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json new file mode 100644 index 0000000..04e026b --- /dev/null +++ b/infra/otel/grafana/dashboards/messenger.json @@ -0,0 +1,174 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": {"type": "grafana", "uid": "-- Grafana --"}, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 0}, + "id": 1, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_route}}", "refId": "A"}], + "title": "API requests per second", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 12, "y": 0}, + "id": 2, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",http_route!~\"/health/.*\"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m])), 1e-9)", "legendFormat": "5xx / business requests", "refId": "A"}], + "title": "Business API 5xx ratio (5m)", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 7}, + "id": 3, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "API latency percentiles from merged histogram buckets", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 7}, + "id": 4, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "Worker job duration percentiles from histogram buckets", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 15}, + "id": 5, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[5m]))", "legendFormat": "{{outcome}}", "refId": "A"}], + "title": "Jobs processed / failed per second", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 15}, + "id": 6, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_queue_depth)", "legendFormat": "queued attachments", "refId": "A"}], + "title": "Durable attachment backlog", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 15}, + "id": 7, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest queued job", "refId": "A"}], + "title": "Age of oldest queued attachment", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 22}, + "id": 8, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_db_pool_connections_used)", "legendFormat": "used", "refId": "A"}, {"expr": "max(messenger_db_pool_requests_waiting)", "legendFormat": "waiting", "refId": "B"}], + "title": "Database pool use / waiters", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 8, "y": 22}, + "id": 9, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "busy / capacity", "refId": "A"}], + "title": "Worker saturation", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 16, "y": 22}, + "id": 10, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_process_memory_usage)", "legendFormat": "RSS", "refId": "A"}], + "title": "Process memory", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 7, "w": 8, "x": 0, "y": 29}, + "id": 11, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_process_cpu_utilization)", "legendFormat": "CPU / core", "refId": "A"}], + "title": "Process CPU utilization", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 7, "w": 16, "x": 8, "y": 29}, + "id": 12, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachments_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95 time to ready", "refId": "A"}], + "title": "Durable acceptance to attachment ready", + "type": "timeseries" + }, + { + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 36}, + "id": 13, + "options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": true, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true}, + "targets": [{"expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", "refId": "A"}], + "title": "Structured API / worker logs", + "type": "logs" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0.9, "max": 1}, "overrides": []}, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 44}, + "id": 14, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}}, + "targets": [{"expr": "messenger:api_availability:ratio7d", "legendFormat": "7-day availability; objective 99%", "refId": "A"}], + "title": "API SLI: 7-day rolling availability (99% SLO)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["messenger", "observability"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timezone": "browser", + "title": "Messenger observability", + "uid": "messenger-observability", + "version": 1 +} diff --git a/infra/otel/grafana/provisioning/dashboards/dashboards.yml b/infra/otel/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..4db9d1f --- /dev/null +++ b/infra/otel/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: Messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: true + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/otel/grafana/provisioning/datasources/datasources.yml b/infra/otel/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..1aa2627 --- /dev/null +++ b/infra/otel/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,39 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://prometheus:9090 + isDefault: true + editable: false + - name: Loki + uid: loki + type: loki + access: proxy + url: https://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id":"([a-f0-9]+)"' + url: "$${__value.raw}" + datasourceUid: tempo + editable: false + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus + editable: false diff --git a/infra/otel/loki.yml b/infra/otel/loki.yml new file mode 100644 index 0000000..454a129 --- /dev/null +++ b/infra/otel/loki.yml @@ -0,0 +1,37 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + http_tls_config: + cert_file: /run/secrets/loki_tls_cert + key_file: /run/secrets/loki_tls_key + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/otel/prometheus-web.yml b/infra/otel/prometheus-web.yml new file mode 100644 index 0000000..53fed1a --- /dev/null +++ b/infra/otel/prometheus-web.yml @@ -0,0 +1,3 @@ +tls_server_config: + cert_file: /run/secrets/prometheus_tls_cert + key_file: /run/secrets/prometheus_tls_key diff --git a/infra/otel/prometheus.yml b/infra/otel/prometheus.yml new file mode 100644 index 0000000..6932fe9 --- /dev/null +++ b/infra/otel/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: otel-collector + scheme: https + tls_config: + ca_file: /run/secrets/ca_crt + cert_file: /run/secrets/collector_prometheus_client_cert + key_file: /run/secrets/collector_prometheus_client_key + server_name: otel-collector + static_configs: + - targets: ["otel-collector:9464"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/infra/otel/tempo.yml b/infra/otel/tempo.yml new file mode 100644 index 0000000..62497d3 --- /dev/null +++ b/infra/otel/tempo.yml @@ -0,0 +1,36 @@ +server: + http_listen_port: 3200 + http_tls_config: + cert_file: /run/secrets/tempo_tls_cert + key_file: /run/secrets/tempo_tls_key + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: /run/secrets/tempo_tls_cert + key_file: /run/secrets/tempo_tls_key + http: + endpoint: 0.0.0.0:4318 + tls: + cert_file: /run/secrets/tempo_tls_cert + key_file: /run/secrets/tempo_tls_key + +ingester: + max_block_duration: 5m + trace_idle_period: 10s + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..b6e6cd1 --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,3 @@ +local all all trust +hostssl all all all scram-sha-256 +hostnossl all all all reject diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..f5431a5 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..aae5c1a --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idempotency_key), + CONSTRAINT idempotency_method_post CHECK (method = 'POST'), + CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..a4178a7 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,59 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank; + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + staging_key text NOT NULL UNIQUE, + job_id uuid NOT NULL UNIQUE, + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + lease_until timestamptz, + status varchar(16) NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'), + error_code varchar(64), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + attempt integer NOT NULL CHECK (attempt >= 1), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + UNIQUE (job_id, attempt) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, created_at) + WHERE published_at IS NULL; + +CREATE TABLE reconciliation_incidents ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')), + first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, incident_type) +); diff --git a/migrations/004_pipeline_replay.sql b/migrations/004_pipeline_replay.sql new file mode 100644 index 0000000..424dcc5 --- /dev/null +++ b/migrations/004_pipeline_replay.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachment_outbox + DROP CONSTRAINT attachment_outbox_job_id_fkey; + +CREATE TABLE attachment_pipeline_events ( + id bigserial PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + prior_job_id uuid NOT NULL, + job_id uuid NOT NULL, + prior_version integer NOT NULL CHECK (prior_version >= 1), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + action varchar(16) NOT NULL CHECK (action = 'replay'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_pipeline_events_attachment_created_idx + ON attachment_pipeline_events (attachment_id, created_at); diff --git a/migrations/005_observability_context.sql b/migrations/005_observability_context.sql new file mode 100644 index 0000000..bd52f66 --- /dev/null +++ b/migrations/005_observability_context.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_outbox + ADD COLUMN traceparent varchar(55), + ADD COLUMN tracestate varchar(512); diff --git a/migrations/006_login_rate_limits.sql b/migrations/006_login_rate_limits.sql new file mode 100644 index 0000000..12a25f7 --- /dev/null +++ b/migrations/006_login_rate_limits.sql @@ -0,0 +1,7 @@ +CREATE TABLE login_rate_limits ( + username_key bytea PRIMARY KEY, + window_started_at timestamptz NOT NULL, + attempts integer NOT NULL CHECK (attempts > 0) +); + +CREATE INDEX login_rate_limits_window_idx ON login_rate_limits (window_started_at); diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103dd..f7e2263 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ -# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. +# Development-only schema validation and dependency auditing tools. openapi-spec-validator==0.7.2 +pip-audit==2.10.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65ddcfa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 +boto3==1.35.99 +Pillow==13.0.0 +pika==1.3.2 +opentelemetry-api==1.45.2 +opentelemetry-sdk==1.45.2 +opentelemetry-exporter-otlp-proto-http==1.45.2 +opentelemetry-instrumentation-psycopg==0.66b2 +opentelemetry-instrumentation-botocore==0.66b2 +psutil==6.1.1 +argon2-cffi==23.1.0 diff --git a/scripts/bootstrap-openbao.py b/scripts/bootstrap-openbao.py new file mode 100644 index 0000000..5b91b54 --- /dev/null +++ b/scripts/bootstrap-openbao.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from pathlib import Path +import ssl +from typing import Optional +import urllib.error +import urllib.request + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +ROLE_DIR = SECRET_DIR / "openbao" +SERVICES = { + "api": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + "worker": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + "publisher": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + "migrate": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), + "reconciler": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"), +} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Provision scoped OpenBao AppRoles for this local lab.") + parser.add_argument("--root-token-file", required=True, help="Operator-only file outside version control") + parser.add_argument("--ca-file", default=str(SECRET_DIR / "pki" / "ca.crt")) + parser.add_argument("--address", default="https://localhost:8200") + args = parser.parse_args() + root_token = Path(args.root_token_file).read_text(encoding="utf-8").strip() + if not root_token: + raise SystemExit("Root token file is empty.") + ROLE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) + context = ssl.create_default_context(cafile=args.ca_file) + + def request(method: str, path: str, payload: Optional[dict] = None) -> dict: + headers = {"X-Vault-Token": root_token, "Content-Type": "application/json"} + body = json.dumps(payload).encode("utf-8") if payload is not None else None + req = urllib.request.Request( + args.address.rstrip("/") + "/v1/" + path, + data=body, + headers=headers, + method=method, + ) + try: + with urllib.request.urlopen(req, context=context, timeout=8) as response: + content = response.read(1024 * 1024) + return json.loads(content) if content else {} + except urllib.error.HTTPError as exc: + raise RuntimeError(f"OpenBao API request failed with HTTP {exc.code}") from None + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc: + raise RuntimeError(f"OpenBao API request failed ({type(exc).__name__})") from None + + mounts = request("GET", "sys/mounts") + if "secret/" not in mounts.get("data", {}): + request("POST", "sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}) + auth = request("GET", "sys/auth") + if "approle/" not in auth.get("data", {}): + request("POST", "sys/auth/approle", {"type": "approle"}) + + local_values = { + "DATABASE_URL": (SECRET_DIR / "database_url").read_text(encoding="utf-8").strip(), + "WORKER_DATABASE_URL": (SECRET_DIR / "worker_database_url").read_text(encoding="utf-8").strip(), + "BROKER_URL": "", + "S3_ACCESS_KEY_ID": (SECRET_DIR / "s3_access_key_id").read_text(encoding="utf-8").strip(), + "S3_SECRET_ACCESS_KEY": (SECRET_DIR / "s3_secret_access_key").read_text(encoding="utf-8").strip(), + } + service_s3_files = { + "api": ("s3_access_key_id", "s3_secret_access_key"), + **{ + service: (f"{service}_s3_access_key_id", f"{service}_s3_secret_access_key") + for service in ("worker", "publisher", "migrate", "reconciler") + }, + } + + for service, keys in SERVICES.items(): + if service in ("worker", "publisher"): + local_values["BROKER_URL"] = ( + SECRET_DIR / f"broker_{service}_url" + ).read_text(encoding="utf-8").strip() + policy_name = f"messenger-{service}" + policy = ( + f'path "secret/data/messenger/{service}" {{\n' + ' capabilities = ["read"]\n' + '}\n' + ) + request("PUT", f"sys/policies/acl/{policy_name}", {"policy": policy}) + request( + "POST", + f"auth/approle/role/{service}", + { + "token_policies": [policy_name], + "token_ttl": "15m", + "token_max_ttl": "1h", + "secret_id_ttl": "24h", + "secret_id_num_uses": 0, + }, + ) + service_values = dict(local_values) + access_key_file, secret_key_file = service_s3_files[service] + service_values["S3_ACCESS_KEY_ID"] = ( + SECRET_DIR / access_key_file + ).read_text(encoding="utf-8").strip() + service_values["S3_SECRET_ACCESS_KEY"] = ( + SECRET_DIR / secret_key_file + ).read_text(encoding="utf-8").strip() + if service == "worker": + service_values["DATABASE_URL"] = local_values["WORKER_DATABASE_URL"] + request( + "POST", + f"secret/data/messenger/{service}", + {"data": {key: service_values[key] for key in keys}}, + ) + role = request("GET", f"auth/approle/role/{service}/role-id") + role_id = role.get("data", {}).get("role_id") + secret = request("POST", f"auth/approle/role/{service}/secret-id", {}) + secret_id = secret.get("data", {}).get("secret_id") + if not role_id or not secret_id: + raise RuntimeError(f"OpenBao did not return credentials for the {service} role") + for suffix, value in (("role_id", role_id), ("secret_id", secret_id)): + path = ROLE_DIR / f"{service}_{suffix}" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + + print("OpenBao policies, scoped AppRoles and service credentials provisioned; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-pki.py b/scripts/bootstrap-pki.py new file mode 100644 index 0000000..bc35743 --- /dev/null +++ b/scripts/bootstrap-pki.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +PKI_DIR = ROOT / "secrets" / "pki" + + +def openssl(*args: str) -> None: + subprocess.run( + ["openssl", *args], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def create_leaf( + work: Path, + name: str, + subject: str, + sans: str, + extended_usage: str, + ca_key: Path, + ca_cert: Path, +) -> tuple[Path, Path]: + key = PKI_DIR / f"{name}.key" + cert = PKI_DIR / f"{name}.crt" + request = work / f"{name}.csr" + extensions = work / f"{name}.ext" + openssl( + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(key), + "-out", + str(request), + "-subj", + subject, + ) + extensions.write_text( + "basicConstraints=critical,CA:FALSE\n" + "keyUsage=critical,digitalSignature,keyEncipherment\n" + f"extendedKeyUsage={extended_usage}\n" + f"subjectAltName={sans}\n", + encoding="ascii", + ) + openssl( + "x509", + "-req", + "-in", + str(request), + "-CA", + str(ca_cert), + "-CAkey", + str(ca_key), + "-CAcreateserial", + "-out", + str(cert), + "-days", + "397", + "-sha256", + "-extfile", + str(extensions), + ) + os.chmod(key, 0o600) + os.chmod(cert, 0o644) + return cert, key + + +def bundle(cert: Path, key: Path, output: Path) -> None: + output.write_bytes(cert.read_bytes() + key.read_bytes()) + os.chmod(output, 0o600) + + +def main() -> None: + if PKI_DIR.exists(): + raise SystemExit("PKI files already exist; rotate them with an explicit rollover procedure.") + if not shutil.which("openssl"): + raise SystemExit("OpenSSL is required to generate local certificates.") + + PKI_DIR.mkdir(mode=0o700, parents=True) + ca_key = PKI_DIR / "ca.key" + ca_cert = PKI_DIR / "ca.crt" + openssl( + "req", + "-x509", + "-newkey", + "rsa:3072", + "-nodes", + "-keyout", + str(ca_key), + "-out", + str(ca_cert), + "-days", + "3650", + "-sha256", + "-subj", + "/CN=mtusi2026lab7-local-ca", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ) + os.chmod(ca_key, 0o600) + os.chmod(ca_cert, 0o644) + + with tempfile.TemporaryDirectory(prefix="lab7-pki-") as directory: + work = Path(directory) + foreign_ca_key = PKI_DIR / "foreign-ca.key" + foreign_ca_cert = PKI_DIR / "foreign-ca.crt" + openssl( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(foreign_ca_key), + "-out", + str(foreign_ca_cert), + "-days", + "365", + "-sha256", + "-subj", + "/CN=mtusi2026lab7-untrusted-ca", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ) + os.chmod(foreign_ca_key, 0o600) + os.chmod(foreign_ca_cert, 0o644) + ingress_cert, ingress_key = create_leaf( + work, + "ingress", + "/CN=localhost", + "DNS:localhost,IP:127.0.0.1", + "serverAuth", + ca_key, + ca_cert, + ) + create_leaf( + work, + "api", + "/CN=api", + "DNS:api,IP:127.0.0.1", + "serverAuth", + ca_key, + ca_cert, + ) + client_cert, client_key = create_leaf( + work, + "lb-client", + "/CN=lab7-load-balancer", + "URI:spiffe://mtusi2026lab7/load-balancer", + "clientAuth", + ca_key, + ca_cert, + ) + create_leaf( + work, + "wrong-client", + "/CN=lab7-untrusted-service", + "URI:spiffe://mtusi2026lab7/other-service", + "clientAuth", + ca_key, + ca_cert, + ) + create_leaf( + work, + "foreign-client", + "/CN=lab7-foreign-service", + "URI:spiffe://mtusi2026lab7/load-balancer", + "clientAuth", + foreign_ca_key, + foreign_ca_cert, + ) + create_leaf( + work, + "openbao", + "/CN=openbao", + "DNS:openbao,DNS:localhost,IP:127.0.0.1", + "serverAuth", + ca_key, + ca_cert, + ) + bundle(ingress_cert, ingress_key, PKI_DIR / "ingress.pem") + bundle(client_cert, client_key, PKI_DIR / "lb-client.pem") + print("Local CA and leaf certificates created under secrets/pki with private permissions.") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-secrets.py b/scripts/bootstrap-secrets.py new file mode 100644 index 0000000..e48a55e --- /dev/null +++ b/scripts/bootstrap-secrets.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +import base64 +import json +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" + + +def main(): + SECRET_DIR.mkdir(mode=0o700, exist_ok=True) + os.chmod(SECRET_DIR, 0o700) + names = ( + "postgres_password", + "database_url", + "worker_db_password", + "worker_database_url", + "s3_access_key_id", + "s3_secret_access_key", + "worker_s3_access_key_id", + "worker_s3_secret_access_key", + "publisher_s3_access_key_id", + "publisher_s3_secret_access_key", + "migrate_s3_access_key_id", + "migrate_s3_secret_access_key", + "reconciler_s3_access_key_id", + "reconciler_s3_secret_access_key", + "s3_admin_access_key_id", + "s3_admin_secret_access_key", + "rabbitmq_password", + "rabbitmq.conf", + "rabbitmq_definitions.json", + "broker_url", + "broker_worker_url", + "broker_publisher_url", + "s3.json", + "s3_iam.json", + "grafana_admin_password", + ) + existing = [name for name in names if (SECRET_DIR / name).exists()] + if existing: + raise SystemExit( + "Secret files already exist; refusing to rotate credentials without a migration: " + + ", ".join(existing) + ) + + db_password = secrets.token_hex(24) + worker_db_password = secrets.token_urlsafe(32) + broker_password = secrets.token_urlsafe(32) + worker_broker_password = secrets.token_urlsafe(32) + publisher_broker_password = secrets.token_urlsafe(32) + values = { + "postgres_password": db_password, + "database_url": ( + f"postgresql://messenger:{db_password}@postgres:5432/messenger" + "?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt" + ), + "worker_db_password": worker_db_password, + "worker_database_url": ( + f"postgresql://messenger_worker:{worker_db_password}@postgres:5432/messenger" + "?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt" + ), + "s3_access_key_id": "mngr" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(40), + "worker_s3_access_key_id": "wrk" + secrets.token_hex(8), + "worker_s3_secret_access_key": secrets.token_urlsafe(40), + "publisher_s3_access_key_id": "pub" + secrets.token_hex(8), + "publisher_s3_secret_access_key": secrets.token_urlsafe(40), + "migrate_s3_access_key_id": "mig" + secrets.token_hex(8), + "migrate_s3_secret_access_key": secrets.token_urlsafe(40), + "reconciler_s3_access_key_id": "rec" + secrets.token_hex(8), + "reconciler_s3_secret_access_key": secrets.token_urlsafe(40), + "s3_admin_access_key_id": "admin" + secrets.token_hex(7), + "s3_admin_secret_access_key": secrets.token_urlsafe(40), + "rabbitmq_password": broker_password, + "broker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f", + "broker_worker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f", + "broker_publisher_url": f"amqps://publisher:{publisher_broker_password}@rabbitmq:5671/%2f", + "grafana_admin_password": secrets.token_urlsafe(32), + } + for name, value in values.items(): + if name.startswith("broker_") and name != "broker_url": + descriptor = os.open(SECRET_DIR / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + continue + path = SECRET_DIR / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + descriptor = os.open( + SECRET_DIR / "rabbitmq.conf", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write("default_user = messenger\n") + output.write(f"default_pass = {broker_password}\n") + output.write("listeners.tcp = none\n") + output.write("listeners.ssl.default = 5671\n") + output.write("ssl_options.cacertfile = /run/secrets/ca_crt\n") + output.write("ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n") + output.write("ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n") + output.write("ssl_options.verify = verify_peer\n") + output.write("ssl_options.fail_if_no_peer_cert = true\n") + output.write("ssl_options.versions.1 = tlsv1.2\n") + output.write("definitions.import_backend = local_filesystem\n") + output.write("definitions.local.path = /run/secrets/rabbitmq_definitions.json\n") + definitions = { + "users": [], + "vhosts": [{"name": "/"}], + "permissions": [], + "queues": [ + { + "name": "attachments.process.dead", + "vhost": "/", + "durable": True, + "auto_delete": False, + "arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1}, + }, + { + "name": "attachments.process.v1", + "vhost": "/", + "durable": True, + "auto_delete": False, + "arguments": { + "x-queue-type": "quorum", + "x-quorum-initial-group-size": 1, + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": "attachments.process.dead", + }, + }, + ], + } + for username, password, configure, write, read in ( + ("worker", worker_broker_password, r"^$", r"^$", r"^attachments\.process\.v1$"), + ("publisher", publisher_broker_password, r"^$", r"^amq\.default$", r"^$"), + ): + salt = secrets.token_bytes(4) + password_hash = __import__("hashlib").sha256(salt + password.encode("utf-8")).digest() + definitions["users"].append({ + "name": username, + "password_hash": __import__("base64").b64encode(salt + password_hash).decode("ascii"), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "", + }) + definitions["permissions"].append({ + "user": username, + "vhost": "/", + "configure": configure, + "write": write, + "read": read, + }) + descriptor = os.open( + SECRET_DIR / "rabbitmq_definitions.json", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(definitions, output, separators=(",", ":")) + output.write("\n") + os.chmod(SECRET_DIR / "rabbitmq_definitions.json", 0o644) + s3_config = { + "identities": [ + { + "name": "bootstrap-admin", + "credentials": [ + { + "accessKey": values["s3_admin_access_key_id"], + "secretKey": values["s3_admin_secret_access_key"], + } + ], + "actions": ["Admin"], + }, + { + "name": "messenger-api", + "credentials": [ + { + "accessKey": values["s3_access_key_id"], + "secretKey": values["s3_secret_access_key"], + } + ], + "actions": [ + "Read:messenger-attachments", + "List:messenger-attachments", + "Read:messenger-variants", + "Write:messenger-staging", + ], + }, + { + "name": "messenger-worker", + "credentials": [{ + "accessKey": values["worker_s3_access_key_id"], + "secretKey": values["worker_s3_secret_access_key"], + }], + "actions": [ + "Read:messenger-staging", + "Write:messenger-staging", + "Write:messenger-attachments", + "Write:messenger-variants", + ], + }, + *[ + { + "name": f"messenger-{service}", + "credentials": [{ + "accessKey": values[f"{service}_s3_access_key_id"], + "secretKey": values[f"{service}_s3_secret_access_key"], + }], + "actions": [], + } + for service in ("publisher", "migrate") + ], + { + "name": "messenger-reconciler", + "credentials": [{ + "accessKey": values["reconciler_s3_access_key_id"], + "secretKey": values["reconciler_s3_secret_access_key"], + }], + "actions": [ + f"{action}:{bucket}" + for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants") + for action in ("Read", "Write", "List") + ], + }, + ] + } + descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(s3_config, output, separators=(",", ":")) + output.write("\n") + iam_config = { + "sts": {"signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")}, + "defaultEffect": "Deny", + } + descriptor = os.open(SECRET_DIR / "s3_iam.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(iam_config, output, separators=(",", ":")) + output.write("\n") + role_dir = SECRET_DIR / "openbao" + role_dir.mkdir(mode=0o700, exist_ok=True) + for service in ("api", "worker", "publisher", "migrate", "reconciler"): + for suffix in ("role_id", "secret_id"): + path = role_dir / f"{service}_{suffix}" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + print("Local secrets generated with owner-only permissions; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/issue-service-pki.py b/scripts/issue-service-pki.py new file mode 100644 index 0000000..56ea4ce --- /dev/null +++ b/scripts/issue-service-pki.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +PKI_DIR = ROOT / "secrets" / "pki" +SERVICES = { + "rabbitmq": ("DNS:rabbitmq", "serverAuth"), + "s3": ("DNS:s3", "serverAuth"), + "postgres": ("DNS:postgres", "serverAuth"), + "otel-collector": ("DNS:otel-collector", "serverAuth"), + "tempo": ("DNS:tempo", "serverAuth"), + "loki": ("DNS:loki", "serverAuth"), + "prometheus": ("DNS:prometheus", "serverAuth"), + "collector-prometheus-client": ("URI:spiffe://mtusi2026lab7/collector", "clientAuth"), + "otel-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"), + "otel-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"), + "otel-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"), + "rabbitmq-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"), + "rabbitmq-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"), + "rabbitmq-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"), + "s3-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"), + "s3-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"), + "s3-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"), + "s3-migrate-client": ("URI:spiffe://mtusi2026lab7/migrate", "clientAuth"), + "s3-reconciler-client": ("URI:spiffe://mtusi2026lab7/reconciler", "clientAuth"), + "s3-bootstrap-client": ("URI:spiffe://mtusi2026lab7/s3-bootstrap", "clientAuth"), + "s3-healthcheck-client": ("URI:spiffe://mtusi2026lab7/s3-healthcheck", "clientAuth"), +} + + +def openssl(*args: str) -> None: + subprocess.run( + ["openssl", *args], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Issue local short-lived mTLS service identities.") + parser.add_argument("--days", type=int, default=30) + args = parser.parse_args() + if not 1 <= args.days <= 90: + parser.error("service certificate lifetime must be between 1 and 90 days") + if not (PKI_DIR / "ca.key").is_file() or not (PKI_DIR / "ca.crt").is_file(): + raise SystemExit("Create the local CA first with scripts/bootstrap-pki.py.") + if not shutil.which("openssl"): + raise SystemExit("OpenSSL is required to issue service certificates.") + missing = { + name: identity for name, identity in SERVICES.items() + if not (PKI_DIR / f"{name}.crt").exists() + } + if not missing: + raise SystemExit("All service certificates already exist; rotate them explicitly.") + + with tempfile.TemporaryDirectory(prefix="lab7-service-pki-") as directory: + work = Path(directory) + for name, (san, usage) in missing.items(): + key = PKI_DIR / f"{name}.key" + cert = PKI_DIR / f"{name}.crt" + csr = work / f"{name}.csr" + extension = work / f"{name}.ext" + openssl( + "req", "-new", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(key), "-out", str(csr), "-subj", f"/CN={name}", + ) + extension.write_text( + "basicConstraints=critical,CA:FALSE\n" + "keyUsage=critical,digitalSignature,keyEncipherment\n" + f"extendedKeyUsage={usage}\n" + f"subjectAltName={san}\n", + encoding="ascii", + ) + openssl( + "x509", "-req", "-in", str(csr), "-CA", str(PKI_DIR / "ca.crt"), + "-CAkey", str(PKI_DIR / "ca.key"), "-CAcreateserial", "-out", str(cert), + "-days", str(args.days), "-sha256", "-extfile", str(extension), + ) + os.chmod(key, 0o600) + os.chmod(cert, 0o644) + print(f"Issued {len(missing)} service certificates with {args.days}-day validity.") + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare-secure-secrets.py b/scripts/prepare-secure-secrets.py new file mode 100644 index 0000000..c6387e0 --- /dev/null +++ b/scripts/prepare-secure-secrets.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import json +import os +from pathlib import Path +import secrets +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" + + +def read(name: str) -> str: + return (SECRET_DIR / name).read_text(encoding="utf-8").strip() + + +def write(name: str, content: str, replace: bool = False) -> None: + path = SECRET_DIR / name + flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if replace else os.O_EXCL) + descriptor = os.open(path, flags, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(content) + if not content.endswith("\n"): + output.write("\n") + + +def rabbit_user(name: str, password: str) -> dict: + salt = secrets.token_bytes(4) + digest = hashlib.sha256(salt + password.encode("utf-8")).digest() + return { + "name": name, + "password_hash": base64.b64encode(salt + digest).decode("ascii"), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "", + } + + +def main() -> None: + os.chmod(SECRET_DIR, 0o700) + db_parts = urlsplit(read("database_url")) + query = dict(parse_qsl(db_parts.query, keep_blank_values=True)) + query.update({"sslmode": "verify-full", "sslrootcert": "/run/secrets/ca_crt"}) + write( + "database_url", + urlunsplit((db_parts.scheme, db_parts.netloc, db_parts.path, urlencode(query), "")), + replace=True, + ) + worker_password_path = SECRET_DIR / "worker_db_password" + if not worker_password_path.exists(): + write("worker_db_password", secrets.token_urlsafe(32)) + worker_password = read("worker_db_password") + write( + "worker_database_url", + urlunsplit( + ( + db_parts.scheme, + f"messenger_worker:{worker_password}@{db_parts.hostname}:{db_parts.port}", + db_parts.path, + urlencode(query), + "", + ) + ), + replace=(SECRET_DIR / "worker_database_url").exists(), + ) + + service_s3_credentials = {} + for service in ("worker", "publisher", "migrate", "reconciler"): + access_key_file = f"{service}_s3_access_key_id" + secret_key_file = f"{service}_s3_secret_access_key" + if not (SECRET_DIR / access_key_file).exists(): + write(access_key_file, service[:3] + secrets.token_hex(8)) + if not (SECRET_DIR / secret_key_file).exists(): + write(secret_key_file, secrets.token_urlsafe(40)) + service_s3_credentials[service] = ( + read(access_key_file), + read(secret_key_file), + ) + + credentials = {} + users = [] + permissions = [] + for service, configure, write_access, read_access in ( + ("worker", r"^$", r"^$", r"^attachments\.process\.v1$"), + ("publisher", r"^$", r"^amq\.default$", r"^$"), + ): + password_file = f"rabbitmq_{service}_password" + password_path = SECRET_DIR / password_file + password = password_path.read_text(encoding="utf-8").strip() if password_path.exists() else secrets.token_urlsafe(32) + if not password_path.exists(): + write(password_file, password) + credentials[service] = password + users.append(rabbit_user(service, password)) + permissions.append({ + "user": service, + "vhost": "/", + "configure": configure, + "write": write_access, + "read": read_access, + }) + write( + f"broker_{service}_url", + f"amqps://{service}:{password}@rabbitmq:5671/%2f", + replace=(SECRET_DIR / f"broker_{service}_url").exists(), + ) + + definitions = { + "users": users, + "vhosts": [{"name": "/"}], + "permissions": permissions, + "queues": [ + { + "name": "attachments.process.dead", + "vhost": "/", + "durable": True, + "auto_delete": False, + "arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1}, + }, + { + "name": "attachments.process.v1", + "vhost": "/", + "durable": True, + "auto_delete": False, + "arguments": { + "x-queue-type": "quorum", + "x-quorum-initial-group-size": 1, + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": "attachments.process.dead", + }, + }, + ], + } + definitions_path = SECRET_DIR / "rabbitmq_definitions.json" + if definitions_path.exists(): + definitions_path.unlink() + write("rabbitmq_definitions.json", json.dumps(definitions, separators=(",", ":"))) + os.chmod(definitions_path, 0o644) + + config = ( + "default_user = messenger\n" + f"default_pass = {read('rabbitmq_password')}\n" + "listeners.tcp = none\n" + "listeners.ssl.default = 5671\n" + "ssl_options.cacertfile = /run/secrets/ca_crt\n" + "ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n" + "ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n" + "ssl_options.verify = verify_peer\n" + "ssl_options.fail_if_no_peer_cert = true\n" + "ssl_options.versions.1 = tlsv1.2\n" + "definitions.import_backend = local_filesystem\n" + "definitions.local.path = /run/secrets/rabbitmq_definitions\n" + ) + write("rabbitmq.conf", config, replace=True) + identities = [ + { + "name": "bootstrap-admin", + "credentials": [{ + "accessKey": read("s3_admin_access_key_id"), + "secretKey": read("s3_admin_secret_access_key"), + }], + "actions": ["Admin"], + }, + { + "name": "messenger-api", + "credentials": [{ + "accessKey": read("s3_access_key_id"), + "secretKey": read("s3_secret_access_key"), + }], + "actions": [ + "Read:messenger-attachments", + "List:messenger-attachments", + "Read:messenger-variants", + "Write:messenger-staging", + ], + }, + { + "name": "messenger-worker", + "credentials": [{ + "accessKey": service_s3_credentials["worker"][0], + "secretKey": service_s3_credentials["worker"][1], + }], + "actions": [ + "Read:messenger-staging", + "Write:messenger-staging", + "Write:messenger-attachments", + "Write:messenger-variants", + ], + }, + *[ + { + "name": f"messenger-{service}", + "credentials": [{ + "accessKey": service_s3_credentials[service][0], + "secretKey": service_s3_credentials[service][1], + }], + "actions": [], + } + for service in ("publisher", "migrate") + ], + { + "name": "messenger-reconciler", + "credentials": [{ + "accessKey": service_s3_credentials["reconciler"][0], + "secretKey": service_s3_credentials["reconciler"][1], + }], + "actions": [ + f"{action}:{bucket}" + for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants") + for action in ("Read", "Write", "List") + ], + }, + ] + write("s3.json", json.dumps({"identities": identities}, separators=(",", ":")), replace=True) + iam_config_path = SECRET_DIR / "s3_iam.json" + if not iam_config_path.exists(): + iam_config = { + "sts": { + "signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii") + }, + "defaultEffect": "Deny", + } + write("s3_iam.json", json.dumps(iam_config, separators=(",", ":"))) + print("TLS endpoints and service-scoped broker credentials prepared locally; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile-loop.sh b/scripts/reconcile-loop.sh new file mode 100755 index 0000000..cbd430f --- /dev/null +++ b/scripts/reconcile-loop.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +interval=${RECONCILE_INTERVAL_SECONDS:-900} +grace=${RECONCILE_GRACE_SECONDS:-3600} + +case "$interval" in + ''|*[!0-9]*) + echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +case "$grace" in + ''|*[!0-9]*) + echo "RECONCILE_GRACE_SECONDS must be an integer" >&2 + exit 2 + ;; +esac +if [ "$interval" -lt 60 ]; then + echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2 + exit 2 +fi +if [ "$grace" -lt 60 ]; then + echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2 + exit 2 +fi + +while : +do + python3 src/reconcile.py --grace-seconds "$grace" --apply + sleep "$interval" +done diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..5b09cc1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +docker compose --env-file .env restart postgres rabbitmq s3-core s3 + +for service in postgres rabbitmq s3-core s3; do + ready=0 + attempt=0 + while [ "$attempt" -lt 90 ]; do + ready=1 + containers=$(docker compose --env-file .env ps -q "$service") + [ -n "$containers" ] || ready=0 + for container in $containers; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}running{{end}}' "$container") + [ "$status" = healthy ] || ready=0 + done + [ "$ready" -eq 1 ] && break + attempt=$((attempt + 1)) + sleep 1 + done + [ "$ready" -eq 1 ] || { + printf 'Service %s did not become healthy after restart.\n' "$service" >&2 + exit 1 + } +done + +docker compose --env-file .env restart api publisher worker reconciler +printf '%s\n' 'Restarted API and application stores without changing volumes.' diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..26ebae1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +matches=0 +target_container= +for container in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + matches=$((matches + 1)) + target_container=$container + fi +done + +if [ "$matches" -ne 1 ]; then + printf '%s\n' 'Could not resolve TARGET_INSTANCE_ID to exactly one API container.' >&2 + exit 1 +fi + +docker stop --time 2 "$target_container" >/dev/null diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..e6bed6a 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,31 @@ -# Модель угроз — заполнить +# Модель угроз — lab7 ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +Активы: пароли пользователей, hashes сессий, сообщения и вложения, database/object-store/broker credentials, service identities, CA private key, OpenBao root token и unseal shares, telemetry. Акторы: внешний клиент, пользователь — участник чата, посторонний пользователь, скомпрометированный application service и локальный оператор. + +Внешний атакующий может отправлять HTTPS запросы к loopback-published ingress. Пользователь действует только от своего Bearer session. Скомпрометированная служба получает только AppRole credential и секретный KV path, смонтированные этой службе. Оператор управляет PKI и OpenBao init/unseal. Модель не включает компрометацию Docker host, операционной системы или operator workstation. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +Внешняя граница — клиент → HAProxy по HTTPS. Внутренний API boundary — HAProxy → API по mTLS с проверкой CA chain и allowlisted URI SAN. OpenBao доступен по TLS и выдаёт статические secrets сервисным AppRole identities. Схема сетевых связей и протоколы приведены в [матрице доверия](../contracts/trust-matrix.md). | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | - -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +| Клиент в сети пытается перехватить пароль при входе или Bearer token | Credentials передаются только аутентифицированному endpoint | HTTPS ingress, CA и hostname validation; открытый HTTP endpoint не опубликован | `make test-security` с корректным и пустым trust store | Внешний endpoint использует локальную учебную CA, которую клиент должен получить доверенным способом | +| Клиент с сертификатом, подписанным доверенной CA, но с URI SAN другой службы, обращается к API | Только LB identity может установить соединение к API listener | mTLS + точное сравнение `spiffe://mtusi2026lab7/load-balancer` | `make test-internal-mtls` с identity `other-service` | Срок leaf ограничен 397 днями; автоматическая ротация не настроена | +| Клиент без сертификата или с cert чужой CA пытается обойти ingress и обратиться к API listener | API не обслуживает неподтверждённое внутреннее соединение | `CERT_REQUIRED`, CA verification, API порт не опубликован на host | `make test-internal-mtls` без cert и с foreign CA; прямого host-порта нет | Проверка выполняется внутри контейнерной сети лабораторного Compose | +| Пользователь вне чата запрашивает историю, metadata или content чужой attachment | Только член чата получает доступ к сообщениям и вложениям | Bearer session, ownership/membership checks, одинаковый 404 для чужих ресурсов | `make test` black-box сценарий ACL | Контракт не включает администрирование/редактирование состава участников | +| При утечке PostgreSQL snapshot атакующий подбирает пароль | Пароль не хранится plaintext и имеет per-user salt | Argon2id `m=19456 KiB, t=2, p=1`; сессионные токены хранятся как SHA-256 hashes | `make test-primitives` | Параметры следует повторно измерить на целевом hardware | +| Password spray пытается обойти лимит через несколько API-реплик и usernames | Серия неуспешных входов должна давать ограниченный ответ | PostgreSQL хранит fixed-window counters по SHA-256(username) и trusted source IP; sixth per-user attempt или 61st per-IP attempt в окне возвращает 429/Retry-After | `make test-login-limit` проверяет per-user threshold | Порог per-IP равен 60 попыткам за минуту | +| Скомпрометированный API/worker запрашивает operator или чужие сервисные secrets | Application identity не получает root token/unseal shares и чужие KV paths | AppRole per service, KV v2 path-specific read policy, short TTL | `make test-openbao-acl` подтверждает собственный path и 403 для остальных сервисов | Поведение после истечения TTL не тестировалось; KV values статические | +| Скомпрометированный worker пытается читать аккаунты/originals, перечислять buckets или публиковать свои jobs | Worker не получает DB users, S3 List/read originals/variants или broker publish privileges | Отдельная PostgreSQL-роль; S3 read/write на staging bucket, write на originals/variants; AMQP read только process queue | `make test-worker-isolation` проверяет grants, `AccessDenied` и broker `403`; `make test-worker-rotation` отзывает прежний SecretID и восстанавливает queued attachment | Проверен один worker и один queued recovery scenario на локальном Compose стенде | +| Ошибка приложения/инструментации записывает token или тело сообщения в telemetry | Credentials и пользовательские данные не экспортируются как telemetry attributes | OTel instrumentation задаёт низкокардинальные route attributes; проверка кодом/конфигурацией | `make test` не является проверкой утечек telemetry; отдельный grep/inspection не приложен | Отдельный telemetry redaction test не запускался | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +Оператор создаёт локальную CA и leaf сертификаты с `scripts/bootstrap-pki.py`; сертификаты и ключи лежат в ignored `secrets/pki/`. Root token/unseal shares создаются оператором при init и не передаются bootstrap скрипту через CLI arguments или app containers; `bootstrap-openbao.py` читает root token из локального файла, пишет policies и создаёт AppRole credentials под `secrets/openbao/`. Скрипт не печатает значения. + +Ingress/API/OpenBao leaf certificates выпускаются на 397 дней; CA — на 3650 дней. Автоматический renewal/CA rollover не реализован. OpenBao AppRole SecretIDs имеют TTL 24 часа, сервисные tokens — 15 минут (max 1 час); приложение аутентифицируется и получает KV values при запуске. Ротация SecretID требует operator provisioning и перезапуска соответствующей службы. OpenBao state сохраняется в named volume; после рестарта оператор unseal-ит сервис по необходимости. + +Runtime TLS проверен для внутренних сервисных каналов и telemetry backends; AppRole access isolation, worker ACL и восстановление после SecretID rotation зафиксированы отдельными сценариями. Автоматический rollover leaf-сертификатов не включён; оператор управляет их сроком и заменой. diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..5bc8da3 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..8e26858 --- /dev/null +++ b/src/app.py @@ -0,0 +1,77 @@ +import hashlib +import hmac +import logging +import os +import signal +import ssl +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .security import allowed_client_identity +from .store import Store +from .telemetry import initialize, start_runtime_metrics + +LOGGER = logging.getLogger("messenger") + + +class MutualTLSServer(ThreadingHTTPServer): + def __init__(self, address, handler): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(os.environ["TLS_CERT_FILE"], os.environ["TLS_KEY_FILE"]) + context.load_verify_locations(cafile=os.environ["TLS_CA_FILE"]) + context.verify_mode = ssl.CERT_REQUIRED + self.tls_context = context + client_cert_pem = open(os.environ["TLS_CLIENT_CERT_FILE"], encoding="ascii").read() + client_cert_der = ssl.PEM_cert_to_DER_cert(client_cert_pem) + self.client_certificate_fingerprint = hashlib.sha256(client_cert_der).digest() + super().__init__(address, handler) + + def get_request(self): + raw_socket, address = super().get_request() + try: + connection = self.tls_context.wrap_socket(raw_socket, server_side=True) + peer_certificate = connection.getpeercert(binary_form=True) + fingerprint = hashlib.sha256(peer_certificate or b"").digest() + if ( + not peer_certificate + or not hmac.compare_digest(fingerprint, self.client_certificate_fingerprint) + or not allowed_client_identity(connection.getpeercert()) + ): + connection.close() + raise ssl.SSLError("client certificate identity is not authorized") + return connection, address + except (OSError, ssl.SSLError): + raw_socket.close() + raise + + +def run_server(): + store = Store() + store.open() + start_runtime_metrics(store) + try: + handler = partial(MessengerHandler, store=store) + server = MutualTLSServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-api")) + run_server() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 0000000..3904b6f --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +import logging +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger("messenger.s3.bootstrap") + + +def migrate_legacy_namespaces(client, source_bucket: str, targets: dict[str, str]) -> int: + copied = 0 + for prefix, target_bucket in targets.items(): + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=source_bucket, + Prefix=prefix, + ) + for page in pages: + for item in page.get("Contents", []): + key = item["Key"] + try: + target = client.head_object(Bucket=target_bucket, Key=key) + except ClientError as exc: + if exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 404: + raise + source = client.get_object(Bucket=source_bucket, Key=key) + body = source["Body"] + try: + content = body.read() + finally: + body.close() + if len(content) != item["Size"]: + raise RuntimeError(f"Legacy object read size mismatch for {key}") + client.put_object( + Bucket=target_bucket, + Key=key, + Body=content, + ContentLength=len(content), + ContentType=source.get("ContentType", "application/octet-stream"), + Metadata=source.get("Metadata", {}), + ) + target = client.head_object(Bucket=target_bucket, Key=key) + copied += 1 + if ( + target["ContentLength"] != item["Size"] + or target.get("ETag") != item.get("ETag") + ): + raise RuntimeError(f"Legacy object copy verification failed for {key}") + return copied + + +def secret(name: str) -> str: + direct = os.getenv(name) + if direct: + return direct + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError(f"Set {name} or {name}_FILE") + + +def main(): + client = boto3.client( + "s3", + endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name="us-east-1", + aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"), + aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"), + verify=os.getenv("S3_CA_FILE", True), + config=Config( + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 3}, + client_cert=(os.getenv("S3_CLIENT_CERT_FILE"), os.getenv("S3_CLIENT_KEY_FILE")), + ), + ) + buckets = ( + os.getenv("S3_BUCKET", "messenger-attachments"), + os.getenv("S3_STAGING_BUCKET", "messenger-staging"), + os.getenv("S3_VARIANTS_BUCKET", "messenger-variants"), + ) + for bucket in buckets: + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + try: + client.create_bucket(Bucket=bucket) + except ClientError as create_error: + if create_error.response.get("Error", {}).get("Code") != "BucketAlreadyOwnedByYou": + raise + client.put_bucket_acl(Bucket=bucket, ACL="private") + copied = migrate_legacy_namespaces( + client, + buckets[0], + { + "staging/": buckets[1], + "variants/": buckets[2], + }, + ) + LOGGER.info("private S3 buckets bootstrapped; legacy objects copied=%d", copied) + + +if __name__ == "__main__": + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + main() diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..5034438 --- /dev/null +++ b/src/config.py @@ -0,0 +1,126 @@ +import json +import os +import re +import ssl +import socket +import uuid +from pathlib import Path +import threading +import urllib.error +import urllib.request + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10")) +UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60")) +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_STAGING_BUCKET = os.getenv("S3_STAGING_BUCKET", "messenger-staging") +S3_VARIANTS_BUCKET = os.getenv("S3_VARIANTS_BUCKET", "messenger-variants") + + +_BAO_LOCK = threading.Lock() +_BAO_CACHE = {} + + +def _read_secret_file(path_variable: str) -> str: + file_path = os.getenv(path_variable) + if not file_path: + raise RuntimeError(f"Set {path_variable}") + return Path(file_path).read_text(encoding="utf-8").strip() + + +def _bao_secrets() -> dict: + with _BAO_LOCK: + if "values" in _BAO_CACHE: + return _BAO_CACHE["values"] + address = os.getenv("BAO_ADDR") + ca_file = os.getenv("BAO_CACERT") + role_id = _read_secret_file("BAO_ROLE_ID_FILE") + secret_id = _read_secret_file("BAO_SECRET_ID_FILE") + secret_path = os.getenv("BAO_SECRET_PATH") + if not address or not ca_file or not secret_path: + raise RuntimeError("OpenBao service authentication requires BAO_ADDR, BAO_CACERT and BAO_SECRET_PATH") + context = ssl.create_default_context(cafile=ca_file) + + def request(path: str, payload: dict, token: str = "", method: str = "") -> dict: + headers = {"Content-Type": "application/json"} + if token: + headers["X-Vault-Token"] = token + data = json.dumps(payload).encode("utf-8") if payload else None + req = urllib.request.Request( + address.rstrip("/") + "/v1/" + path, + data=data, + headers=headers, + method=method or None, + ) + try: + with urllib.request.urlopen(req, context=context, timeout=5) as response: + return json.loads(response.read(1024 * 1024)) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError, ValueError) as exc: + raise RuntimeError(f"OpenBao request failed ({type(exc).__name__})") from None + + auth = request("auth/approle/login", {"role_id": role_id, "secret_id": secret_id}) + token = auth.get("auth", {}).get("client_token") + if not token: + raise RuntimeError("OpenBao AppRole login returned no service token") + response = request(f"secret/data/{secret_path.lstrip('/')}", {}, token, "GET") + values = response.get("data", {}).get("data") + if not isinstance(values, dict): + raise RuntimeError("OpenBao returned an invalid secret payload") + _BAO_CACHE["values"] = values + return values + + +def secret_value(name: str, default: str = "") -> str: + if os.getenv("BAO_ROLE_ID_FILE") or os.getenv("BAO_ROLE_ID"): + values = _bao_secrets() + if name in values and isinstance(values[name], str): + return values[name] + raise RuntimeError(f"OpenBao path does not contain {name}") + value = os.getenv(name) + if value: + return value + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + if default: + return default + raise RuntimeError(f"Set {name} or {name}_FILE") + + +S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY") +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if IDEMPOTENCY_TTL_SECONDS < 86400: + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") +if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024: + raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB") +if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000: + raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million") +if MAX_CONCURRENT_UPLOADS < 1: + raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive") +if UPLOAD_TIMEOUT_SECONDS < 1: + raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..de78501 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,26 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass + + +class AttachmentNotReadyError(Exception): + pass + + +class ObjectMissingError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 0000000..5923627 --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,653 @@ +import base64 +import hashlib +import json +import logging +import threading +import time +import tempfile +import uuid +import warnings +import zlib +from http.server import BaseHTTPRequestHandler +from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit +import unicodedata + +import psycopg +from opentelemetry import propagate +from opentelemetry.instrumentation.utils import suppress_instrumentation +from opentelemetry.trace import SpanKind, Status, StatusCode +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +from psycopg_pool import PoolTimeout + +from .config import ( + IDEMPOTENCY_KEY_RE, + INSTANCE_ID, + MAX_CONCURRENT_UPLOADS, + MAX_IMAGE_PIXELS, + MAX_INFLIGHT_REQUESTS, + MAX_REQUEST_BODY_BYTES, + MAX_UPLOAD_BYTES, + UPLOAD_TIMEOUT_SECONDS, + USER_RE, +) +from .errors import ( + ApiError, + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .security import PasswordHashCapacityError, hash_password, isoformat, valid_uuid +from .store import Store +from .telemetry import REQUEST_ID, record_http_request, tracer + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() +UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS) +IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1) + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def setup(self): + super().setup() + self.connection.settimeout(15) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + path = urlsplit(self.path).path + route = self._route_template(path) + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + parent_context = propagate.extract({key.lower(): value for key, value in self.headers.items()}) + started = time.perf_counter() + token = REQUEST_ID.set(request_id) + self._response_status = 500 + if route.startswith("/health/"): + try: + self._dispatch_request(method, request_id) + finally: + record_http_request(method, route, self._response_status, time.perf_counter() - started) + REQUEST_ID.reset(token) + return + with tracer().start_as_current_span( + f"{method} {route}", + context=parent_context, + kind=SpanKind.SERVER, + attributes={ + "http.request.method": method, + "http.route": route, + "http.request_id": request_id, + }, + ) as span: + try: + self._dispatch_request(method, request_id) + finally: + duration = time.perf_counter() - started + span.set_attribute("http.response.status_code", self._response_status) + if self._response_status >= 500: + span.set_status(Status(StatusCode.ERROR)) + record_http_request(method, route, self._response_status, duration) + REQUEST_ID.reset(token) + + @staticmethod + def _route_template(path: str) -> str: + if path in ("/health/live", "/health/ready", "/api/v1/users", "/api/v1/auth/sessions", + "/api/v1/auth/sessions/current", "/api/v1/users/me", "/api/v1/chats"): + return path + if path.startswith("/api/v1/chats/") and path.endswith("/attachments"): + return "/api/v1/chats/{chat_id}/attachments" + if path.startswith("/api/v1/chats/") and path.endswith("/messages"): + return "/api/v1/chats/{chat_id}/messages" + if path.startswith("/api/v1/chats/"): + return "/api/v1/chats/{chat_id}" + if path.startswith("/api/v1/attachments/") and path.endswith("/content"): + return "/api/v1/attachments/{attachment_id}/content" + if path.startswith("/api/v1/attachments/"): + return "/api/v1/attachments/{attachment_id}" + return "unmatched" + + def _dispatch_request(self, method: str, request_id: str): + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + with suppress_instrumentation(): + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "A required storage service is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST": + chat_id = path.rsplit("/", 2)[1] + self._handle_upload_attachment(chat_id, query, request_id) + elif path.startswith("/api/v1/attachments/") and method == "GET": + attachment_path = path.removeprefix("/api/v1/attachments/") + if attachment_path.endswith("/content"): + attachment_id = attachment_path.removesuffix("/content") + self._handle_download_attachment(attachment_id, query, request_id) + else: + self._handle_get_attachment(attachment_path, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except PasswordHashCapacityError: + self._send_error( + 503, + "unavailable", + "Password verification capacity is busy", + request_id, + {"Retry-After": "1"}, + ) + except IdempotencyConflictError: + self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id) + except AttachmentNotReadyError: + self._send_error(409, "conflict", "Attachment is not ready", request_id) + except ObjectMissingError: + LOGGER.error("attachment object missing for request %s", request_id) + self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id) + except (BotoCoreError, ClientError): + LOGGER.warning("object storage unavailable for request %s", request_id) + self._send_error(503, "unavailable", "Object storage is unavailable", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + self._set_response_status(499) + self.close_connection = True + LOGGER.info("client disconnected before response for request %s", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + @staticmethod + def _trace_context(): + carrier = {} + propagate.inject(carrier) + return {"traceparent": carrier.get("traceparent"), "tracestate": carrier.get("tracestate")} + + def _set_response_status(self, status: int) -> None: + self._response_status = status + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + if not USER_RE.fullmatch(username) or not 12 <= len(password) <= 128: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + source_ip = self.headers.get("X-Real-IP", self.client_address[0]) + retry_after = self.store.consume_login_attempt(username, source_ip) + if retry_after is not None: + raise ApiError( + 429, + "rate_limited", + "Too many login attempts", + {"Retry-After": str(retry_after)}, + ) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + self.store.clear_login_attempts(username) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self._set_response_status(204) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + idempotency_key = self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body: + raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids") + text = body["text"] + attachment_ids = body.get("attachment_ids", []) + if ( + not isinstance(text, str) + or len(text) > 4000 + or not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + or (not text.strip() and not attachment_ids) + ): + raise ApiError(400, "invalid_request", "Invalid message text") + attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids] + canonical_body = json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + message = self.store.create_idempotent_message( + chat_id, + actor["id"], + text, + attachment_ids, + idempotency_key, + canonical_body, + ) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]: + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + length = self.headers.get("Content-Length") + if length is None or self.headers.get("Transfer-Encoding"): + raise ApiError(400, "invalid_request", "A bounded Content-Length is required") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size <= 0: + raise ApiError(415, "unsupported_media_type", "Image is empty") + if size > MAX_UPLOAD_BYTES: + self.close_connection = True + remaining = min(size, MAX_UPLOAD_BYTES + 1) + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + break + self.connection.settimeout(min(15, remaining_time)) + try: + chunk = self.rfile.read(min(64 * 1024, remaining)) + except (TimeoutError, OSError): + LOGGER.info("stopped bounded drain of oversized upload") + break + if not chunk: + break + remaining -= len(chunk) + raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit") + content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + digest = hashlib.sha256() + bytes_read = 0 + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + try: + while bytes_read < size: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise ApiError(408, "invalid_request", "Image upload timed out") + self.connection.settimeout(min(15, remaining_time)) + chunk = self.rfile.read(min(64 * 1024, size - bytes_read)) + if not chunk: + raise ApiError(400, "invalid_request", "Incomplete image body") + content.write(chunk) + digest.update(chunk) + bytes_read += len(chunk) + except (TimeoutError, OSError): + content.close() + raise ApiError(400, "invalid_request", "Incomplete image body") from None + + try: + content.seek(0) + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + if actual_type is None or actual_type != declared_type: + raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type") + width, height = image.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") + image.verify() + content.seek(0) + with IMAGE_DECODE_GUARD: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + image.load() + content.seek(0) + return content, actual_type, size, digest.hexdigest() + except ApiError: + content.close() + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + content.close() + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error): + content.close() + raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None + + @staticmethod + def _valid_filename(filename: str) -> bool: + return ( + 1 <= len(filename) <= 255 + and not any(char in ("/", "\\", "\x00") for char in filename) + and not any(unicodedata.category(char) == "Cc" for char in filename) + ) + + def _handle_upload_attachment( + self, chat_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + if not self.store.can_access_chat(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Chat not found") + filenames = query.get("filename", []) + if len(filenames) != 1 or not self._valid_filename(filenames[0]): + raise ApiError(400, "invalid_request", "Invalid filename") + if not UPLOAD_GUARD.acquire(blocking=False): + raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted") + try: + content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type()) + try: + attachment = self.store.create_attachment( + chat_id, + actor["id"], + filenames[0], + content_type, + content, + size, + digest, + **self._trace_context(), + ) + finally: + content.close() + finally: + UPLOAD_GUARD.release() + if attachment is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(202, attachment, request_id) + + def _handle_get_attachment(self, attachment_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"]) + if attachment is None: + raise ApiError(404, "not_found", "Attachment not found") + self._send_json(200, attachment, request_id) + + def _handle_download_attachment( + self, attachment_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + variants = query.get("variant", ["original"]) + if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"): + raise ApiError(400, "invalid_request", "Invalid attachment variant") + variant = variants[0] + if variant != "original" and variant not in ("thumbnail", "crop", "optimized"): + raise ApiError(404, "not_found", "Attachment variant not found") + result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant) + if result is None: + raise ApiError(404, "not_found", "Attachment not found") + content_type, content = result + self._set_response_status(200) + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(content))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Cache-Control", "private, no-store") + self.end_headers() + self.wfile.write(content) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + self._set_response_status(status) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..08f5607 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,33 @@ +import json +import os +import ssl + +import pika +from .config import secret_value + +QUEUE_NAME = "attachments.process.v1" +DEAD_QUEUE_NAME = "attachments.process.dead" +LEASE_NETWORK_BUDGET_SECONDS = 150 + + +def broker_url() -> str: + return secret_value("BROKER_URL") + + +def connect() -> pika.BlockingConnection: + parameters = pika.URLParameters(broker_url()) + context = ssl.create_default_context(cafile=os.environ["BROKER_CA_FILE"]) + context.load_cert_chain( + os.environ["BROKER_CLIENT_CERT_FILE"], + os.environ["BROKER_CLIENT_KEY_FILE"], + ) + parameters.ssl_options = pika.SSLOptions(context, parameters.host) + parameters.connection_attempts = 3 + parameters.retry_delay = 1 + parameters.socket_timeout = 5 + parameters.blocked_connection_timeout = 10 + return pika.BlockingConnection(parameters) + + +def encode_job(job: dict) -> bytes: + return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..ef58b0a --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import logging +import os +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + worker_password = Path(os.environ["WORKER_DB_PASSWORD_FILE"]).read_text(encoding="utf-8").strip() + store.configure_worker_role(worker_password) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 0000000..37680f8 --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,101 @@ +import logging +import os +from datetime import datetime +from typing import BinaryIO, Iterator, Optional, Tuple, Union + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from .config import ( + S3_ACCESS_KEY_ID, + S3_BUCKET, + S3_ENDPOINT, + S3_REGION, + S3_SECRET_ACCESS_KEY, + S3_STAGING_BUCKET, + S3_VARIANTS_BUCKET, +) +from .errors import ObjectMissingError + +LOGGER = logging.getLogger("messenger.s3") + + +def bucket_for_key(key: str) -> str: + if key.startswith("staging/"): + return S3_STAGING_BUCKET + if key.startswith("original/"): + return S3_BUCKET + if key.startswith("variants/"): + return S3_VARIANTS_BUCKET + raise ValueError("Object key is outside configured storage namespaces") + + +class ObjectStore: + def __init__(self): + self.client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + verify=os.getenv("S3_CA_FILE", True), + config=Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + client_cert=( + os.getenv("S3_CLIENT_CERT_FILE"), + os.getenv("S3_CLIENT_KEY_FILE"), + ), + s3={"addressing_style": "path"}, + ), + ) + + def put( + self, + key: str, + content: Union[bytes, BinaryIO], + content_type: str, + size_bytes: Optional[int] = None, + ) -> None: + if size_bytes is None: + size_bytes = len(content) + self.client.put_object( + Bucket=bucket_for_key(key), + Key=key, + Body=content, + ContentLength=size_bytes, + ContentType=content_type, + ) + + def get(self, key: str, max_bytes: int) -> bytes: + try: + response = self.client.get_object(Bucket=bucket_for_key(key), Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("NoSuchKey", "404", "NotFound"): + raise ObjectMissingError from exc + raise + body = response["Body"] + try: + value = body.read(max_bytes + 1) + finally: + body.close() + if len(value) > max_bytes: + raise RuntimeError("Stored object exceeds configured safety limit") + return value + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=bucket_for_key(key), Key=key) + + def ready(self) -> bool: + self.client.head_bucket(Bucket=S3_BUCKET) + return True + + def objects(self) -> Iterator[Tuple[str, datetime]]: + for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET): + pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket) + for page in pages: + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] diff --git a/src/pipeline_versions.py b/src/pipeline_versions.py new file mode 100644 index 0000000..5b87cfa --- /dev/null +++ b/src/pipeline_versions.py @@ -0,0 +1,4 @@ +PIPELINE_QUALITY = { + 1: {"jpeg": 85, "webp": 80}, + 2: {"jpeg": 80, "webp": 75}, +} diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 0000000..d73fb4e --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +import logging +import os +import time +import uuid + +import pika +import psycopg +from opentelemetry import propagate +from opentelemetry.instrumentation.utils import suppress_instrumentation +from opentelemetry.trace import SpanKind + +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, encode_job +from .security import isoformat +from .store import Store +from .telemetry import initialize, tracer + +LOGGER = logging.getLogger("messenger.publisher") + + +def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool: + with store.pool.connection() as conn: + with conn.transaction(): + with suppress_instrumentation(): + row = store.claim_outbox(conn) + if row is None: + return False + payload = { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": isoformat(row["created_at"]), + } + parent_context = propagate.extract( + { + key: row.get(key) + for key in ("traceparent", "tracestate") + if row.get(key) is not None + } + ) + with tracer().start_as_current_span( + "attachment.publish", + context=parent_context, + kind=SpanKind.PRODUCER, + attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "publish"}, + ): + carrier = {} + propagate.inject(carrier) + payload.update(carrier) + channel.basic_publish( + exchange="", + routing_key=QUEUE_NAME, + body=encode_job(payload), + properties=pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + message_id=str(row["id"]), + type="attachment.process.v1", + ), + mandatory=True, + ) + conn.execute( + "UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + return True + + +def main() -> None: + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-publisher")) + store = Store() + store.open() + try: + while True: + connection = None + try: + connection = connect() + channel = connection.channel() + channel.confirm_delivery() + store.recover_expired_attachment_jobs( + int(os.getenv("JOB_MAX_ATTEMPTS", "5")), + int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS, + ) + if not publish_pending(store, channel): + time.sleep(1) + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("outbox publishing cycle failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..421e046 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import logging +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +LOGGER = logging.getLogger("messenger.reconcile") + + +def reconcile( + store: Store, + grace_seconds: int, + apply_changes: bool, + restore_dir: Optional[Path] = None, +): + started = time.monotonic() + metadata = store.reconciliation_snapshot() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + found = set() + orphans = [] + for key, last_modified in store.objects.objects(): + found.add(key) + if key not in metadata and last_modified <= cutoff: + orphans.append(key) + + missing = [ + row for key, row in metadata.items() + if key not in found and row.get("reconcile_type", "original") == "original" + ] + restore_candidates = [] + if restore_dir is not None: + root = restore_dir.resolve(strict=True) + for row in missing: + source = restore_dir / str(row["id"]) + try: + resolved = source.resolve(strict=True) + if ( + resolved.parent != root + or not resolved.is_file() + or resolved.stat().st_size != row["size_bytes"] + ): + continue + content = resolved.read_bytes() + except OSError: + continue + if ( + len(content) == row["size_bytes"] + and hashlib.sha256(content).hexdigest() == row["sha256"].strip() + ): + restore_candidates.append((row, content)) + + restored = 0 + if apply_changes: + for key in orphans: + store.objects.delete(key) + restorable_ids = set() + for row, content in restore_candidates: + store.objects.put(row["object_key"], content, row["content_type"]) + store.clear_missing_object(row["id"]) + restorable_ids.add(row["id"]) + restored += 1 + for row in missing: + if row["id"] not in restorable_ids: + store.record_missing_object(row["id"]) + for key, row in metadata.items(): + if key in found and row.get("reconcile_type", "original") == "original": + store.clear_missing_object(row["id"]) + + unresolved_missing = len(missing) - restored + return { + "mode": "apply" if apply_changes else "dry-run", + "grace_seconds": grace_seconds, + "objects_seen": len(found), + "metadata_seen": len({str(row["id"]) for row in metadata.values()}), + "orphan_objects": len([key for key in found if key not in metadata]), + "stale_orphan_candidates": len(orphans), + "deleted_orphans": len(orphans) if apply_changes else 0, + "missing_objects": len(missing), + "unresolved_missing_objects": unresolved_missing, + "restorable_missing_objects": len(restore_candidates), + "restored_missing_objects": restored, + "missing_incidents_recorded": len(missing) - restored if apply_changes else 0, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + } + + +def main(): + parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects") + parser.add_argument( + "--restore-dir", + type=Path, + help="optional trusted backup directory with exact original bytes named by attachment UUID", + ) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error("--grace-seconds must be at least 60") + + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + store = Store() + store.open() + try: + result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir) + print(json.dumps(result, sort_keys=True)) + if result["unresolved_missing_objects"]: + LOGGER.error( + "%s database records reference missing S3 objects; metadata was retained for manual recovery", + result["unresolved_missing_objects"], + ) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/replay_attachment.py b/src/replay_attachment.py new file mode 100644 index 0000000..62ba8bf --- /dev/null +++ b/src/replay_attachment.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import argparse +import json +import uuid + +from .store import Store + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Explicitly replay a failed attachment or start a newer pipeline version." + ) + parser.add_argument("--attachment-id", required=True, type=uuid.UUID) + parser.add_argument( + "--pipeline-version", + required=True, + type=int, + help="pipeline version implemented by the deployed worker", + ) + args = parser.parse_args() + if args.pipeline_version < 1: + parser.error("--pipeline-version must be positive") + + store = Store() + store.open() + try: + result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version) + print(json.dumps(result, sort_keys=True)) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..2cdf529 --- /dev/null +++ b/src/security.py @@ -0,0 +1,134 @@ +import base64 +import hashlib +import hmac +import os +import secrets +import threading +import uuid +from contextlib import contextmanager +from datetime import datetime, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from argon2.low_level import Type + + +_ARGON2_MEMORY_KIB = int(os.getenv("ARGON2_MEMORY_KIB", "19456")) +_ARGON2_ITERATIONS = int(os.getenv("ARGON2_ITERATIONS", "2")) +_ARGON2_PARALLELISM = int(os.getenv("ARGON2_PARALLELISM", "1")) +_PASSWORD_HASH_CONCURRENCY = int(os.getenv("PASSWORD_HASH_CONCURRENCY", "4")) +_PASSWORD_HASH_WAIT_SECONDS = float(os.getenv("PASSWORD_HASH_WAIT_SECONDS", "0.1")) +if ( + _ARGON2_MEMORY_KIB < 19456 + or _ARGON2_MEMORY_KIB > 65536 + or not 2 <= _ARGON2_ITERATIONS <= 4 + or not 1 <= _ARGON2_PARALLELISM <= 4 +): + raise RuntimeError("Argon2id settings must be 19456..65536 KiB, t=2..4, and p=1..4") +if not 1 <= _PASSWORD_HASH_CONCURRENCY <= 4 or _PASSWORD_HASH_WAIT_SECONDS <= 0: + raise RuntimeError("Password hash concurrency must be 1..4 with a positive wait timeout") + +_PASSWORD_HASHER = PasswordHasher( + time_cost=_ARGON2_ITERATIONS, + memory_cost=_ARGON2_MEMORY_KIB, + parallelism=_ARGON2_PARALLELISM, + type=Type.ID, +) +_PASSWORD_HASH_SLOTS = threading.BoundedSemaphore(_PASSWORD_HASH_CONCURRENCY) + + +class PasswordHashCapacityError(RuntimeError): + pass + + +@contextmanager +def _password_hash_slot(): + if not _PASSWORD_HASH_SLOTS.acquire(timeout=_PASSWORD_HASH_WAIT_SECONDS): + raise PasswordHashCapacityError("password hashing capacity is temporarily exhausted") + try: + yield + finally: + _PASSWORD_HASH_SLOTS.release() + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + with _password_hash_slot(): + return _PASSWORD_HASHER.hash(password) + + +def _verify_password(password: str, stored: str) -> bool: + try: + if stored.startswith("$argon2id$"): + parts = stored.split("$") + if len(parts) != 6 or parts[2] != "v=19": + return False + parameters = dict(part.split("=", 1) for part in parts[3].split(",")) + memory, iterations, parallelism = ( + int(parameters["m"]), + int(parameters["t"]), + int(parameters["p"]), + ) + if not (19456 <= memory <= 65536 and 2 <= iterations <= 4 and 1 <= parallelism <= 4): + return False + return _PASSWORD_HASHER.verify(stored, password) + # Existing lab6 accounts can log in once and be upgraded to Argon2id. + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except ( + InvalidHashError, + VerificationError, + VerifyMismatchError, + ValueError, + TypeError, + UnicodeError, + IndexError, + KeyError, + ): + return False + + +def verify_password(password: str, stored: str) -> bool: + with _password_hash_slot(): + return _verify_password(password, stored) + + +def password_hash_needs_upgrade(stored: str) -> bool: + if not stored.startswith("$argon2id$"): + return True + try: + return _PASSWORD_HASHER.check_needs_rehash(stored) + except (InvalidHashError, VerificationError): + return True + + +def allowed_client_identity(cert: dict) -> bool: + return ("URI", "spiffe://mtusi2026lab7/load-balancer") in cert.get("subjectAltName", ()) diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..60fd3d5 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..5f523e6 --- /dev/null +++ b/src/store.py @@ -0,0 +1,934 @@ +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import secrets +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, BinaryIO, Dict, List, Optional, Tuple + +import redis +from psycopg import sql +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + IDEMPOTENCY_TTL_SECONDS, + SESSION_TTL_SECONDS, + secret_value, +) +from .errors import ( + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .object_store import ObjectStore +from .pipeline_versions import PIPELINE_QUALITY +from .security import ( + hash_password, + isoformat, + password_hash_needs_upgrade, + utc_now, + valid_uuid, + verify_password, +) + +LOGGER = logging.getLogger("messenger.store") +_DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(32)) + + +def database_url() -> str: + return secret_value("DATABASE_URL") + + +class Store: + """PostgreSQL metadata, S3 objects, and optional Redis session cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + self.objects = ObjectStore() + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + self.objects.ready() + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + def configure_worker_role(self, password: str) -> None: + with self.pool.connection() as conn: + with conn.transaction(): + exists = conn.execute( + "SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker'" + ).fetchone() + if exists: + statement = sql.SQL("ALTER ROLE messenger_worker LOGIN PASSWORD {}").format( + sql.Literal(password) + ) + else: + statement = sql.SQL("CREATE ROLE messenger_worker LOGIN PASSWORD {}").format( + sql.Literal(password) + ) + conn.execute(statement) + conn.execute("GRANT CONNECT ON DATABASE messenger TO messenger_worker") + conn.execute("GRANT USAGE ON SCHEMA public TO messenger_worker") + conn.execute( + "GRANT SELECT (id, job_id, pipeline_version, attempt, status, lease_until, " + "staging_key, object_key, content_type, size_bytes, sha256) ON attachments " + "TO messenger_worker" + ) + conn.execute( + "GRANT UPDATE (status, lease_until, updated_at, variants, error_code, attempt) " + "ON attachments TO messenger_worker" + ) + conn.execute( + "GRANT INSERT (id, job_id, attachment_id, pipeline_version, attempt, available_at, " + "traceparent, tracestate) ON attachment_outbox TO messenger_worker" + ) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + "attachment_ids": attachment_ids or [], + } + + @staticmethod + def attachment(row: Dict[str, Any]) -> Dict[str, Any]: + variants = row.get("variants") or [] + public_variants = [ + { + name: variant[name] + for name in ("name", "content_type", "size_bytes", "width", "height", "sha256") + } + for variant in variants + ] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"].strip(), + "status": row.get("status", "queued"), + "variants": public_variants, + "error_code": row.get("error_code"), + "created_at": isoformat(row["created_at"]), + "updated_at": isoformat(row["updated_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + stored_hash = row["password_hash"] if row else _DUMMY_PASSWORD_HASH + if not verify_password(password, stored_hash) or not row: + return None + if password_hash_needs_upgrade(stored_hash): + upgraded_hash = hash_password(password) + with self.pool.connection() as conn: + conn.execute( + "UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s", + (upgraded_hash, row["id"], stored_hash), + ) + return self.user(row) + + def consume_login_attempt(self, username: str, source_ip: str) -> Optional[int]: + user_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest() + ip_key = hashlib.sha256(b"ip\0" + source_ip.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + def bump(key: bytes) -> Dict[str, Any]: + return conn.execute( + """ + INSERT INTO login_rate_limits (username_key, window_started_at, attempts) + VALUES (%s, clock_timestamp(), 1) + ON CONFLICT (username_key) DO UPDATE SET + attempts = CASE + WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute' + THEN 1 ELSE login_rate_limits.attempts + 1 + END, + window_started_at = CASE + WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute' + THEN clock_timestamp() ELSE login_rate_limits.window_started_at + END + RETURNING attempts, + GREATEST(1, ceil(extract(epoch FROM + (window_started_at + interval '1 minute' - clock_timestamp()))))::integer AS retry_after + """, + (key,), + ).fetchone() + + user_row = bump(user_key) + ip_row = bump(ip_key) + conn.execute( + "DELETE FROM login_rate_limits WHERE window_started_at < clock_timestamp() - interval '1 hour'" + ) + retries = [] + if user_row["attempts"] > 5: + retries.append(user_row["retry_after"]) + if ip_row["attempts"] > 60: + retries.append(ip_row["retry_after"]) + return max(retries) if retries else None + + def clear_login_attempts(self, username: str) -> None: + username_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest() + with self.pool.connection() as conn: + conn.execute("DELETE FROM login_rate_limits WHERE username_key = %s", (username_key,)) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_idempotent_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: List[str], + idempotency_key: str, + canonical_body: str, + ) -> Optional[Dict[str, Any]]: + request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() + lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") + lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + if attachment_ids: + attachments = conn.execute( + "SELECT id, status FROM attachments " + "WHERE chat_id = %s AND id = ANY(%s) FOR SHARE", + (chat_id, attachment_ids), + ).fetchall() + if len(attachments) != len(attachment_ids): + return None + if any(item["status"] == "failed" for item in attachments): + raise AttachmentNotReadyError + + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = conn.execute( + "SELECT request_hash, response, expires_at " + "FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " + "FOR UPDATE", + (sender_id, chat_id, idempotency_key), + ).fetchone() + if existing and existing["expires_at"] > utc_now(): + if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): + raise IdempotencyConflictError + return existing["response"] + if existing: + conn.execute( + "DELETE FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", + (sender_id, chat_id, idempotency_key), + ) + + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, ordinal) " + "VALUES (%s, %s, %s)", + (row["id"], attachment_id, ordinal), + ) + message = self.message(row, attachment_ids) + conn.execute( + "INSERT INTO idempotency_records " + "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " + "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False, separators=(",", ":")), + utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), + ), + ) + return message + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + ids_by_message = {} + if rows: + links = conn.execute( + "SELECT message_id, attachment_id FROM message_attachments " + "WHERE message_id = ANY(%s) ORDER BY message_id, ordinal", + ([row["id"] for row in rows],), + ).fetchall() + for link in links: + ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"])) + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor + + def can_access_chat(self, chat_id: str, user_id: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() is not None + + def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + content: BinaryIO, + size_bytes: int, + digest: str, + traceparent: Optional[str] = None, + tracestate: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + if not self.can_access_chat(chat_id, owner_id): + return None + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"original/{attachment_id}" + content.seek(0) + self.objects.put(staging_key, content, content_type, size_bytes) + try: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO attachments " + "(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "object_key, staging_key, job_id) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " + "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "status, variants, error_code, created_at, updated_at", + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + digest, + object_key, + staging_key, + job_id, + ), + ).fetchone() + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " + "VALUES (%s, %s, %s, 1, 1, %s, %s)", + (uuid.uuid4(), job_id, attachment_id, traceparent, tracestate), + ) + return self.attachment(row) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "attachment staging compensation failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + + def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + return self.attachment(row) if row else None + + def download_attachment( + self, attachment_id: str, user_id: str, variant: str = "original" + ) -> Optional[Tuple[str, bytes]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + if not row: + return None + if row["status"] != "ready": + raise AttachmentNotReadyError + if variant == "original": + key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"] + else: + match = next( + (item for item in (row["variants"] or []) if item["name"] == variant), + None, + ) + if match is None: + return None + key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"] + return content_type, self.objects.get(key, size_bytes) + + def claim_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int + ) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, " + "object_key, content_type, size_bytes, sha256 " + "FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + not row + or str(row["job_id"]) != job_id + or row["pipeline_version"] != pipeline_version + or row["status"] in ("ready", "failed") + or row["attempt"] != attempt + ): + return None + if row["status"] == "processing" and row["lease_until"] is not None: + if row["lease_until"] > utc_now(): + return None + return conn.execute( + "UPDATE attachments SET status = 'processing', " + "lease_until = clock_timestamp() + (%s * interval '1 second'), " + "updated_at = clock_timestamp() " + "WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, " + "size_bytes, sha256, attempt, pipeline_version", + (lease_seconds, attachment_id), + ).fetchone() + + def complete_attachment_job( + self, + attachment_id: str, + job_id: str, + pipeline_version: int, + attempt: int, + variants: List[Dict[str, Any]], + ) -> bool: + with self.pool.connection() as conn: + with conn.transaction(): + updated = conn.execute( + "UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, " + "lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s " + "AND status = 'processing' AND lease_until > clock_timestamp()", + ( + json.dumps(variants, separators=(",", ":")), + attachment_id, + job_id, + pipeline_version, + attempt, + ), + ) + return updated.rowcount == 1 + + def fail_attachment_job( + self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, + max_attempts: int, error_code: str, retry_delay_seconds: Optional[float], + traceparent: Optional[str] = None, tracestate: Optional[str] = None, + ) -> None: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "SELECT status, attempt FROM attachments " + "WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE", + (attachment_id, job_id, pipeline_version), + ).fetchone() + if not row or row["status"] != "processing" or row["attempt"] != attempt: + return + if retry_delay_seconds is not None and attempt < max_attempts: + next_attempt = attempt + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, error_code, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) " + "VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) " + "ON CONFLICT (job_id, attempt) DO NOTHING", + ( + uuid.uuid4(), job_id, attachment_id, pipeline_version, + next_attempt, retry_delay_seconds, traceparent, tracestate, + ), + ) + else: + final_code = "retry_exhausted" if attempt >= max_attempts else error_code + conn.execute( + "UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, " + "updated_at = clock_timestamp() WHERE id = %s", + (final_code, attachment_id), + ) + + def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]: + if pipeline_version not in PIPELINE_QUALITY: + raise ValueError("pipeline version is not implemented by this deployment") + with self.pool.connection() as conn: + current = conn.execute( + "SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, " + "content_type, size_bytes, sha256 FROM attachments WHERE id = %s", + (attachment_id,), + ).fetchone() + if current is None: + raise ValueError("attachment not found") + if pipeline_version < current["pipeline_version"]: + raise ValueError("pipeline version cannot move backwards") + if pipeline_version == current["pipeline_version"] and current["status"] != "failed": + raise ValueError("same-version replay is allowed only for failed attachments") + if current["status"] not in ("queued", "processing", "ready", "failed"): + raise ValueError("attachment cannot be replayed in its current state") + + try: + content = self.objects.get(current["object_key"], current["size_bytes"]) + except ObjectMissingError as object_error: + try: + content = self.objects.get(current["staging_key"], current["size_bytes"]) + except ObjectMissingError: + raise object_error + if ( + len(content) != current["size_bytes"] + or hashlib.sha256(content).hexdigest() != current["sha256"].strip() + ): + raise ValueError("stored original does not match attachment metadata") + + prior_job_id = str(current["job_id"]) + next_job_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}/{next_job_id}" + self.objects.put(staging_key, content, current["content_type"], len(content)) + try: + with self.pool.connection() as conn: + with conn.transaction(): + locked = conn.execute( + "SELECT job_id, pipeline_version, attempt, status FROM attachments " + "WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if ( + locked is None + or str(locked["job_id"]) != prior_job_id + or locked["pipeline_version"] != current["pipeline_version"] + or locked["attempt"] != current["attempt"] + or locked["status"] != current["status"] + ): + raise RuntimeError("attachment changed while preparing replay") + conn.execute( + "UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, " + "status = 'queued', staging_key = %s, variants = '[]'::jsonb, " + "error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() " + "WHERE id = %s", + (next_job_id, pipeline_version, staging_key, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, 1)", + (uuid.uuid4(), next_job_id, attachment_id, pipeline_version), + ) + conn.execute( + "INSERT INTO attachment_pipeline_events " + "(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) " + "VALUES (%s, %s, %s, %s, %s, 'replay')", + ( + attachment_id, + prior_job_id, + next_job_id, + current["pipeline_version"], + pipeline_version, + ), + ) + except Exception: + try: + self.objects.delete(staging_key) + except Exception as cleanup_error: + LOGGER.error( + "replay staging cleanup failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + return { + "attachment_id": str(attachment_id), + "previous_job_id": prior_job_id, + "job_id": str(next_job_id), + "pipeline_version": pipeline_version, + "attempt": 1, + "status": "queued", + } + + def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + "SELECT id, job_id, pipeline_version, attempt FROM attachments " + "WHERE status = 'processing' AND lease_until < clock_timestamp() " + "ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100" + ).fetchall() + for row in rows: + if row["attempt"] >= max_attempts: + conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', " + "lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + continue + next_attempt = row["attempt"] + 1 + conn.execute( + "UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, " + "error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s", + (next_attempt, row["id"]), + ) + conn.execute( + "INSERT INTO attachment_outbox " + "(id, job_id, attachment_id, pipeline_version, attempt) " + "VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING", + (uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt), + ) + return len(rows) + + def claim_outbox(self, conn): + return conn.execute( + "SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, " + "o.traceparent, o.tracestate, " + "a.created_at FROM attachment_outbox o " + "JOIN attachments a ON a.id = o.attachment_id " + "WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() " + "ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1" + ).fetchone() + + def queue_snapshot(self) -> Tuple[int, float]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT count(*) AS depth, " + "COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age " + "FROM attachments WHERE status = 'queued'" + ).fetchone() + return int(row["depth"]), float(row["oldest_age"]) + + def reconciliation_snapshot(self): + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, " + "size_bytes, sha256, content_type FROM attachments" + ).fetchall() + expected = {} + for row in rows: + if row["status"] == "ready": + expected[row["object_key"]] = {**row, "reconcile_type": "original"} + for variant in row["variants"] or []: + expected[variant["object_key"]] = { + **row, + "object_key": variant["object_key"], + "size_bytes": variant["size_bytes"], + "sha256": variant["sha256"], + "content_type": variant["content_type"], + "reconcile_type": "variant", + } + elif row["status"] in ("queued", "processing"): + expected[row["staging_key"]] = {**row, "reconcile_type": "staging"} + expected[row["object_key"]] = {**row, "reconcile_type": "inflight"} + for name, extension in ( + ("thumbnail", "jpg"), + ("crop", "jpg"), + ("optimized", "webp"), + ): + key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}" + expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"} + return expected + + def record_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO reconciliation_incidents (attachment_id, incident_type) " + "VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) " + "DO UPDATE SET last_seen_at = clock_timestamp()", + (attachment_id,), + ) + + def clear_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM reconciliation_incidents " + "WHERE attachment_id = %s AND incident_type = 'missing_object'", + (attachment_id,), + ) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..560c0ba --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,300 @@ +import contextvars +import json +import logging +import os +import platform +import socket +import threading +import time +from datetime import datetime, timezone +from typing import Optional + +import psycopg +from opentelemetry import metrics, trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON, ParentBased, TraceIdRatioBased +from opentelemetry.instrumentation.utils import suppress_instrumentation + +REQUEST_ID = contextvars.ContextVar("request_id", default="") +_lock = threading.Lock() +_runtime = { + "pool_used": 0, + "pool_waiting": 0, + "queue_depth": 0, + "queue_oldest_seconds": 0.0, + "worker_busy": 0, + "worker_capacity": 0, + "cpu": 0.0, + "memory": 0, +} +_provider = None +_meter = None +_service_name = "messenger" +_enabled = True +_request_counter = None +_request_duration = None +_job_counter = None +_job_duration = None +_time_to_ready = None +_queue_depth = None +_queue_oldest = None +_pool_used = None +_pool_waiting = None +_worker_busy = None +_worker_capacity = None +_process_cpu = None +_process_memory = None + + +class JsonLogFormatter(logging.Formatter): + converter = time.gmtime + + def format(self, record): + span_context = trace.get_current_span().get_span_context() + document = { + "timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(), + "severity": record.levelname, + "service.name": _service_name, + "event": record.getMessage(), + "request_id": REQUEST_ID.get(), + } + if span_context.is_valid: + document["trace_id"] = format(span_context.trace_id, "032x") + document["span_id"] = format(span_context.span_id, "016x") + if record.exc_info: + document["exception.type"] = record.exc_info[0].__name__ + return json.dumps(document, ensure_ascii=True, separators=(",", ":")) + + +def _sampler(): + if os.getenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") == "always_off": + return ALWAYS_OFF + if os.getenv("OTEL_TRACES_SAMPLER", "") == "always_on": + return ALWAYS_ON + ratio = float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0")) + if not 0 <= ratio <= 1: + raise ValueError("OTEL_TRACES_SAMPLER_ARG must be between 0 and 1") + return ParentBased(TraceIdRatioBased(ratio)) + + +def initialize(service_name: str) -> None: + global _enabled + global _service_name + global _provider, _meter + global _request_counter, _request_duration, _job_counter, _job_duration, _time_to_ready + global _queue_depth, _queue_oldest, _pool_used, _pool_waiting + global _worker_busy, _worker_capacity, _process_cpu, _process_memory + + enabled = os.getenv("OTEL_SDK_DISABLED", "false").lower() != "true" + _enabled = enabled + _service_name = service_name + instance_id = os.getenv("INSTANCE_ID", socket.gethostname()) + resource = Resource.create( + { + "service.name": service_name, + "service.version": os.getenv("SERVICE_VERSION", "lab6"), + "service.instance.id": instance_id, + "deployment.environment.name": os.getenv("APP_ENV", "lab"), + "host.name": socket.gethostname(), + "os.type": platform.system().lower(), + } + ) + if enabled: + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318") + _provider = TracerProvider(resource=resource, sampler=_sampler()) + _provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint.rstrip("/") + "/v1/traces"), + max_queue_size=4096, + max_export_batch_size=512, + schedule_delay_millis=1000, + export_timeout_millis=10000, + ) + ) + trace.set_tracer_provider(_provider) + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint.rstrip("/") + "/v1/metrics"), + export_interval_millis=10000, + export_timeout_millis=5000, + ) + metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[reader])) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint.rstrip("/") + "/v1/logs"), + max_queue_size=4096, + max_export_batch_size=512, + schedule_delay_millis=1000, + export_timeout_millis=10000, + ) + ) + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + else: + trace.set_tracer_provider(TracerProvider(resource=resource, sampler=ALWAYS_OFF)) + metrics.set_meter_provider(MeterProvider(resource=resource)) + otel_handler = None + + root = logging.getLogger() + root.setLevel(os.getenv("LOG_LEVEL", "INFO")) + formatter = JsonLogFormatter() + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + root.handlers[:] = [stream_handler] + if otel_handler is not None: + root.addHandler(otel_handler) + logging.getLogger("pika").setLevel(logging.WARNING) + + if enabled: + try: + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + + BotocoreInstrumentor().instrument() + PsycopgInstrumentor().instrument() + except ImportError as exc: + raise RuntimeError("OTel database and S3 instrumentation dependencies are required") from exc + + _meter = metrics.get_meter("messenger.telemetry") + _request_counter = _meter.create_counter("messenger.http.requests", unit="{request}") + _request_duration = _meter.create_histogram("messenger.http.duration", unit="s") + _job_counter = _meter.create_counter("messenger.jobs.processed", unit="{job}") + _job_duration = _meter.create_histogram("messenger.jobs.duration", unit="s") + _time_to_ready = _meter.create_histogram("messenger.attachments.time_to_ready", unit="s") + + def observe(name, attributes=None): + def callback(_options): + with _lock: + return [metrics.Observation(_runtime[name], attributes=attributes or {})] + + return callback + + _queue_depth = _meter.create_observable_gauge( + "messenger.queue.depth", + callbacks=[observe("queue_depth", {"queue.name": "attachments.process.v1"})], + unit="{job}", + ) + _queue_oldest = _meter.create_observable_gauge( + "messenger.queue.oldest_age", + callbacks=[observe("queue_oldest_seconds", {"queue.name": "attachments.process.v1"})], + unit="s", + ) + _pool_used = _meter.create_observable_gauge( + "messenger.db.pool.connections.used", + callbacks=[observe("pool_used", {"pool.name": "primary"})], + unit="{connection}", + ) + _pool_waiting = _meter.create_observable_gauge( + "messenger.db.pool.requests.waiting", + callbacks=[observe("pool_waiting", {"pool.name": "primary"})], + unit="{request}", + ) + _worker_busy = _meter.create_observable_gauge( + "messenger.worker.busy", callbacks=[observe("worker_busy")], unit="{worker}" + ) + _worker_capacity = _meter.create_observable_gauge( + "messenger.worker.capacity", callbacks=[observe("worker_capacity")], unit="{worker}" + ) + _process_cpu = _meter.create_observable_gauge( + "messenger.process.cpu.utilization", callbacks=[observe("cpu")], unit="1" + ) + _process_memory = _meter.create_observable_gauge( + "messenger.process.memory.usage", callbacks=[observe("memory")], unit="By" + ) + + +def tracer(): + return trace.get_tracer("messenger.application") + + +def record_http_request(method: str, route: str, status: int, duration: float) -> None: + attributes = { + "http.request.method": method, + "http.route": route, + "status_class": f"{status // 100}xx", + } + if _request_counter is not None: + _request_counter.add(1, attributes) + _request_duration.record(duration, attributes) + if _enabled and not route.startswith("/health/"): + logging.getLogger("messenger.http").info( + "http request completed method=%s route=%s status_class=%s", + method, + route, + f"{status // 100}xx", + ) + + +def record_job(outcome: str, duration: float, time_to_ready: Optional[float] = None) -> None: + attributes = {"job.type": "attachment.process.v1", "outcome": outcome} + if _job_counter is not None: + _job_counter.add(1, attributes) + _job_duration.record(duration, attributes) + if time_to_ready is not None: + _time_to_ready.record(time_to_ready, {"outcome": outcome}) + + +def update_runtime(**values) -> None: + with _lock: + _runtime.update(values) + + +def start_runtime_metrics( + store, worker_capacity: int = 0, include_queue_snapshot: bool = True +) -> Optional[threading.Thread]: + if not _enabled: + return None + + def refresh(): + try: + import psutil + + process = psutil.Process() + process.cpu_percent(interval=None) + while True: + try: + stats = store.pool.get_stats() + if include_queue_snapshot: + with suppress_instrumentation(): + queue_depth, oldest_age = store.queue_snapshot() + cpu = process.cpu_percent(interval=1) / 100.0 + memory = process.memory_info().rss + runtime = dict( + pool_used=max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)), + pool_waiting=stats.get("requests_waiting", 0), + worker_capacity=worker_capacity, + cpu=cpu, + memory=memory, + ) + if include_queue_snapshot: + runtime.update(queue_depth=queue_depth, queue_oldest_seconds=oldest_age) + update_runtime(**runtime) + except (psycopg.Error, OSError, RuntimeError, ValueError): + logging.getLogger("messenger.telemetry").exception("runtime metric sample failed") + time.sleep(9) + except (OSError, RuntimeError, ValueError): + logging.getLogger("messenger.telemetry").exception("runtime metric sampler stopped") + raise + + thread = threading.Thread(target=refresh, name="runtime-metrics", daemon=True) + thread.start() + return thread + + +def worker_started() -> None: + with _lock: + _runtime["worker_busy"] += 1 + + +def worker_stopped() -> None: + with _lock: + _runtime["worker_busy"] = max(0, _runtime["worker_busy"] - 1) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..3524700 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +import hashlib +import io +import json +import logging +import multiprocessing +import os +import random +import time +import uuid +import warnings +import zlib +from datetime import datetime, timezone +from typing import Callable, Optional + +import pika +import psycopg +from opentelemetry import propagate +from opentelemetry.trace import SpanKind, Status, StatusCode +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, ImageOps, UnidentifiedImageError + +from .config import MAX_IMAGE_PIXELS +from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect +from .pipeline_versions import PIPELINE_QUALITY +from .store import Store +from .telemetry import ( + initialize, + record_job, + start_runtime_metrics, + tracer, + worker_started, + worker_stopped, +) + +LOGGER = logging.getLogger("messenger.worker") + +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1")) + + +class InvalidAttachment(Exception): + pass + + +class UnsupportedPipeline(InvalidAttachment): + pass + + +def _rgb_image(content: bytes) -> Image.Image: + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(content)) as source: + if source.format not in ("PNG", "JPEG"): + raise InvalidAttachment("unsupported_image") + width, height = source.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise InvalidAttachment("image_too_large") + source.load() + image = ImageOps.exif_transpose(source) + try: + if "A" in image.getbands() or "transparency" in image.info: + with image.convert("RGBA") as rgba: + background = Image.new("RGB", rgba.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + finally: + if image is not source: + image.close() + except InvalidAttachment: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError, + UnidentifiedImageError, ValueError, zlib.error) as exc: + raise InvalidAttachment("invalid_image") from exc + + +def _encode(image: Image.Image, image_format: str, **options) -> bytes: + output = io.BytesIO() + image.save(output, format=image_format, **options) + return output.getvalue() + + +def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1): + quality = PIPELINE_QUALITY.get(pipeline_version) + if quality is None: + raise UnsupportedPipeline("unsupported_pipeline_version") + image = _rgb_image(content) + try: + variants = [] + thumbnail = image.copy() + try: + thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes)) + finally: + thumbnail.close() + + crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + try: + crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True) + variants.append(("crop", "image/jpeg", 128, 128, crop_bytes)) + finally: + crop.close() + + optimized = image.copy() + try: + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6) + variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes)) + finally: + optimized.close() + + result = [] + for name, content_type, width, height, data in variants: + extension = "webp" if content_type == "image/webp" else "jpg" + key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}" + result.append( + { + "name": name, + "content_type": content_type, + "size_bytes": len(data), + "width": width, + "height": height, + "sha256": hashlib.sha256(data).hexdigest(), + "object_key": key, + "content": data, + } + ) + return result + finally: + image.close() + + +class ProcessingTimeout(TimeoutError): + pass + + +def _variant_process(connection, attachment_id, content, pipeline_version, builder): + try: + connection.send(("ok", builder(attachment_id, content, pipeline_version))) + except InvalidAttachment as exc: + connection.send(("invalid", str(exc))) + except Exception as exc: + connection.send(("error", type(exc).__name__)) + finally: + connection.close() + + +def _build_variants_bounded( + attachment_id: str, + content: bytes, + pipeline_version: int, + timeout_seconds: int, + builder=build_variants, +): + if timeout_seconds < 1: + raise ValueError("timeout_seconds must be positive") + context = multiprocessing.get_context("spawn") + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_variant_process, + args=(sender, attachment_id, content, pipeline_version, builder), + ) + process.start() + sender.close() + try: + if not receiver.poll(timeout_seconds): + process.terminate() + process.join(timeout=2) + if process.is_alive(): + process.kill() + process.join() + raise ProcessingTimeout("image processing exceeded its time budget") + try: + result, value = receiver.recv() + except EOFError as exc: + raise RuntimeError("image processing process exited without a result") from exc + process.join(timeout=2) + if process.is_alive(): + process.terminate() + process.join() + if result == "invalid": + raise InvalidAttachment(value) + if result == "error": + raise RuntimeError(f"image processing failed in subprocess ({value})") + return value + finally: + receiver.close() + if process.is_alive(): + process.terminate() + process.join() + + +def _job(payload: bytes): + value = json.loads(payload) + required = { + "schema_version", "job_id", "type", "attachment_id", + "pipeline_version", "attempt", "created_at", + } + if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}): + raise ValueError("invalid job fields") + if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1": + raise ValueError("invalid job schema") + uuid.UUID(value["job_id"]) + uuid.UUID(value["attachment_id"]) + if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1: + raise ValueError("invalid pipeline version") + if not isinstance(value["attempt"], int) or value["attempt"] < 1: + raise ValueError("invalid job attempt") + return value + + +def _process( + store: Store, + job: dict, + fault_injector: Optional[Callable[[str], None]] = None, +) -> str: + def inject(point: str) -> None: + if fault_injector is not None: + fault_injector(point) + + attachment_id = job["attachment_id"] + row = store.claim_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS, + ) + if row is None: + LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"]) + return "skipped" + + try: + inject("after_claim") + with tracer().start_as_current_span("attachment.read_staging"): + content = store.objects.get(row["staging_key"], row["size_bytes"]) + if hashlib.sha256(content).hexdigest() != row["sha256"].strip(): + raise InvalidAttachment("checksum_mismatch") + variants = _build_variants_bounded( + attachment_id, + content, + job["pipeline_version"], + JOB_TIMEOUT_SECONDS, + ) + inject("before_original_put") + with tracer().start_as_current_span("attachment.write_original"): + store.objects.put(row["object_key"], content, row["content_type"], len(content)) + inject("after_original_put") + persisted = [] + for variant in variants: + data = variant.pop("content") + inject(f"before_variant_put:{variant['name']}") + with tracer().start_as_current_span("attachment.write_variant"): + store.objects.put(variant["object_key"], data, variant["content_type"], len(data)) + inject(f"after_variant_put:{variant['name']}") + persisted.append(variant) + inject("before_commit") + with tracer().start_as_current_span("attachment.commit_ready"): + completed = store.complete_attachment_job( + attachment_id, + job["job_id"], + job["pipeline_version"], + job["attempt"], + persisted, + ) + if not completed: + LOGGER.warning("job lease was superseded before completion: %s", job["job_id"]) + return "skipped" + inject("after_commit") + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id) + LOGGER.info("attachment processing completed: %s", attachment_id) + return "success" + except InvalidAttachment as exc: + carrier = {} + propagate.inject(carrier) + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, str(exc), None, carrier.get("traceparent"), carrier.get("tracestate"), + ) + try: + store.objects.delete(row["staging_key"]) + except (BotoCoreError, ClientError, OSError): + LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id) + LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc) + return "failed" + except Exception as exc: + delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25) + carrier = {} + propagate.inject(carrier) + store.fail_attachment_job( + attachment_id, job["job_id"], job["pipeline_version"], job["attempt"], + MAX_ATTEMPTS, "processing_failed", delay, carrier.get("traceparent"), carrier.get("tracestate"), + ) + LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__) + return "failed" + + +def handle_delivery( + store: Store, + channel: pika.adapters.blocking_connection.BlockingChannel, + method, + body: bytes, + fault_injector: Optional[Callable[[str], None]] = None, +) -> None: + try: + job = _job(body) + except (ValueError, TypeError, json.JSONDecodeError): + LOGGER.exception("invalid broker job rejected") + channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + return + parent_context = propagate.extract( + {key: job[key] for key in ("traceparent", "tracestate") if key in job} + ) + started = time.perf_counter() + created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")) + with tracer().start_as_current_span( + "attachment.process", + context=parent_context, + kind=SpanKind.CONSUMER, + attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "process"}, + ) as span: + worker_started() + try: + outcome = _process(store, job, fault_injector) + if outcome == "failed": + span.set_status(Status(StatusCode.ERROR)) + if outcome != "skipped": + elapsed = time.perf_counter() - started + ready_duration = ( + max(0.0, datetime.now(timezone.utc).timestamp() - created_at.timestamp()) + if outcome == "success" + else None + ) + record_job(outcome, elapsed, ready_duration) + finally: + worker_stopped() + if fault_injector is not None: + fault_injector("before_ack") + channel.basic_ack(delivery_tag=method.delivery_tag) + + +def main() -> None: + if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1: + raise RuntimeError("Worker limits must be positive") + initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-worker")) + store = Store() + store.open() + start_runtime_metrics(store, WORKER_CONCURRENCY, include_queue_snapshot=False) + connection = None + try: + while True: + try: + connection = connect() + channel = connection.channel() + + channel.basic_qos(prefetch_count=WORKER_CONCURRENCY) + + def on_message(ch, method, properties, body): + handle_delivery(store, ch, method, body) + + channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False) + LOGGER.info("worker consuming durable attachment queue") + channel.start_consuming() + except (OSError, pika.exceptions.AMQPError, psycopg.Error): + LOGGER.exception("worker connection failed") + time.sleep(2) + finally: + if connection is not None and connection.is_open: + connection.close() + connection = None + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..1663aaa --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Negative attachment and cross-chat authorization checks against a running lab4 API.""" +import uuid +from pathlib import Path +import sys +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202): + return client.call( + "POST", + "/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""), + expected, + actor=actor, + raw=data, + headers={"Content-Type": content_type}, + schema="Attachment" if expected in (201, 202) else None, + ) + + +def main(): + args = arguments("Attachment upload, size, MIME, and chat authorization checks") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + other_chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + print("Checking accepted fixture upload.", flush=True) + attachment = upload(client, alice, chat["id"], image)["json"] + + print("Checking damaged and spoofed image rejection.", flush=True) + upload(client, alice, chat["id"], b"not an image", expected=415) + upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415) + print("Checking byte and pixel limits.", flush=True) + upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413) + import struct + import zlib + + def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30")) + bad_idat[-3] ^= 0xFF + damaged_idat = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", bytes(bad_idat)) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], damaged_idat, expected=415) + + too_many_pixels = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], too_many_pixels, expected=413) + print("Checking filename and cross-chat authorization.", flush=True) + upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400) + upload(client, eve, chat["id"], image, expected=404) + client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve) + client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + 404, + actor=alice, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": "", "attachment_ids": [attachment["id"]]}, + ) + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=bob, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": ""}, + ) + print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_idempotency_race.py b/student-tests/test_idempotency_race.py new file mode 100644 index 0000000..646869a --- /dev/null +++ b/student-tests/test_idempotency_race.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a concurrent idempotency-key race through the load balancer.""" +from concurrent.futures import ThreadPoolExecutor +import threading +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency integration check") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = "race-" + uuid.uuid4().hex + text = "concurrent-" + uuid.uuid4().hex + count = 12 + barrier = threading.Barrier(count) + + def send_once(_): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=10) + return worker.message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send_once, range(count))) + + instance_ids = {response["headers"]["X-Instance-Id"] for response in responses} + messages = [response["json"] for response in responses] + check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas") + check(all(message == messages[0] for message in messages), "same-key requests returned different messages") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"] + matches = [message for message in page["items"] if message["text"] == text] + check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages") + print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_outbox_recovery_integration.py b/student-tests/test_outbox_recovery_integration.py new file mode 100644 index 0000000..972228b --- /dev/null +++ b/student-tests/test_outbox_recovery_integration.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify an upload committed while the publisher is stopped is processed after restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify committed attachment outbox intent survives publisher downtime") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + compose("stop", "publisher") + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted into queued state") + finally: + compose("start", "publisher") + + ready = client.wait_ready(alice, attachment) + stored_original = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + thumbnail = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + variant = next(item for item in ready["variants"] if item["name"] == "thumbnail") + check(stored_original == original, "recovered job changed original bytes") + check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch") + print( + json.dumps( + { + "result": "PASS", + "accepted_status_before_publisher_restart": attachment["status"], + "final_status": ready["status"], + "attachment_id": attachment["id"], + "original_sha256": hashlib.sha256(stored_original).hexdigest(), + "thumbnail_sha256": variant["sha256"], + "recovered_without_manual_republish": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..0d06e56 --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing.""" +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import uuid +from datetime import datetime, timezone + +os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret") + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +import pika + +from src.publisher import publish_pending +from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery + + +ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001" +JOB_ID = "00000000-0000-4000-8000-000000000002" + + +class SimulatedCrash(BaseException): + pass + + +class FakeObjects: + def __init__(self, content): + self.data = {"staging/" + ATTACHMENT_ID: content} + self.puts = [] + + def get(self, key, max_bytes): + return self.data[key] + + def put(self, key, content, content_type, size_bytes=None): + value = content if isinstance(content, bytes) else content.read() + assert len(value) == size_bytes + self.data[key] = value + self.puts.append((key, hashlib.sha256(value).hexdigest())) + + def delete(self, key): + self.data.pop(key, None) + + +class FakeStore: + def __init__(self, content): + self.objects = FakeObjects(content) + self.row = { + "id": ATTACHMENT_ID, + "job_id": JOB_ID, + "pipeline_version": 1, + "attempt": 1, + "status": "queued", + "lease_until": 0.0, + "staging_key": "staging/" + ATTACHMENT_ID, + "object_key": "original/" + ATTACHMENT_ID, + "content_type": "image/png", + "size_bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + "variants": [], + } + + def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] in ("ready", "failed") + ): + return None + if row["status"] == "processing" and row["lease_until"] > time.monotonic(): + return None + row["status"] = "processing" + row["lease_until"] = time.monotonic() + lease_seconds + return dict(row) + + def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants): + row = self.row + if ( + attachment_id != row["id"] + or job_id != row["job_id"] + or version != row["pipeline_version"] + or attempt != row["attempt"] + or row["status"] != "processing" + or row["lease_until"] <= time.monotonic() + ): + return False + row["status"] = "ready" + row["lease_until"] = 0.0 + row["variants"] = variants + return True + + def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay): + row = self.row + if ( + row["id"] == attachment_id + and row["job_id"] == job_id + and row["pipeline_version"] == version + and row["attempt"] == attempt + and row["status"] == "processing" + ): + row["status"] = "failed" if delay is None else "queued" + row["lease_until"] = 0.0 + row["error_code"] = error_code + if delay is not None: + row["attempt"] += 1 + + def recover_expired(self): + if self.row["status"] == "processing": + self.row["attempt"] += 1 + self.row["status"] = "queued" + self.row["lease_until"] = 0.0 + + def replay(self, version): + row = self.row + previous_job = row["job_id"] + row["job_id"] = str(uuid.uuid4()) + row["pipeline_version"] = version + row["attempt"] = 1 + row["status"] = "queued" + row["lease_until"] = 0.0 + row["variants"] = [] + row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}" + self.objects.put( + row["staging_key"], + self.objects.data[row["object_key"]], + row["content_type"], + row["size_bytes"], + ) + return { + "schema_version": 1, + "job_id": row["job_id"], + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + "previous_job_id": previous_job, + } + + +class FakeChannel: + def __init__(self): + self.acked = [] + self.nacked = [] + + def basic_ack(self, delivery_tag): + self.acked.append(delivery_tag) + + def basic_nack(self, delivery_tag, requeue): + self.nacked.append((delivery_tag, requeue)) + + +class FakeMethod: + def __init__(self, delivery_tag): + self.delivery_tag = delivery_tag + + +def job_body(job_id=JOB_ID, version=1, attempt=1): + return json.dumps( + { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": ATTACHMENT_ID, + "pipeline_version": version, + "attempt": attempt, + "created_at": "2026-01-01T00:00:00Z", + } + ).encode() + + +def crash_at(expected): + def inject(point): + if point == expected: + raise SimulatedCrash(point) + + return inject + + +def slow_builder(attachment_id, content, pipeline_version): + time.sleep(5) + + +def test_duplicate_and_ack_loss(content): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(1), job_body(), + fault_injector=crash_at("before_ack"), + ) + raise AssertionError("fault injection did not crash before ack") + except SimulatedCrash: + pass + assert store.row["status"] == "ready" + first_write_count = len(store.objects.puts) + assert channel.acked == [] + + handle_delivery(store, channel, FakeMethod(2), job_body()) + assert channel.acked == [2] + assert len(store.objects.puts) == first_write_count + assert store.row["status"] == "ready" + return { + "scenario": "duplicate_delivery_after_commit_before_ack", + "deliveries": 2, + "effective_object_writes": first_write_count, + "duplicate_delivery_writes": len(store.objects.puts) - first_write_count, + } + + +def test_crash_before_and_after_side_effect(content, crash_point): + store = FakeStore(content) + channel = FakeChannel() + try: + handle_delivery( + store, channel, FakeMethod(3), job_body(), + fault_injector=crash_at(crash_point), + ) + raise AssertionError(f"fault injection did not crash at {crash_point}") + except SimulatedCrash: + pass + assert channel.acked == [] + store.recover_expired() + retry = job_body(attempt=2) + handle_delivery(store, channel, FakeMethod(4), retry) + assert store.row["status"] == "ready" + assert store.row["attempt"] == 2 + assert channel.acked == [4] + assert store.objects.data["original/" + ATTACHMENT_ID] == content + return { + "scenario": "crash_" + crash_point, + "attempts": store.row["attempt"], + "terminal_status": store.row["status"], + } + + +def test_reordered_attempts(content): + store = FakeStore(content) + store.row["attempt"] = 2 + channel = FakeChannel() + handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2)) + writes_after_current = len(store.objects.puts) + handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1)) + assert store.row["status"] == "ready" + assert len(store.objects.puts) == writes_after_current + assert channel.acked == [5, 6] + return { + "scenario": "out_of_order_attempt_delivery", + "current_attempt": store.row["attempt"], + "stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current, + } + + +def test_stale_pipeline_cannot_publish_newer_result(content): + store = FakeStore(content) + channel = FakeChannel() + replayed = [] + + def supersede_after_first_variant(point): + if point == "after_variant_put:thumbnail" and not replayed: + replayed.append(store.replay(2)) + + handle_delivery( + store, channel, FakeMethod(7), job_body(), + fault_injector=supersede_after_first_variant, + ) + assert store.row["status"] == "queued" + assert store.row["pipeline_version"] == 2 + assert channel.acked == [7] + + current_job = replayed[0] + handle_delivery( + store, + channel, + FakeMethod(8), + job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]), + ) + assert store.row["status"] == "ready" + assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"} + assert channel.acked == [7, 8] + return { + "scenario": "stale_pipeline_after_newer_version", + "current_pipeline_version": store.row["pipeline_version"], + "published_variant_versions": sorted( + {variant["object_key"].split("/")[2] for variant in store.row["variants"]} + ), + } + + +class FakeTransaction: + def __init__(self, store): + self.store = store + + def __enter__(self): + self.published = False + self.store.current_transaction = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None and self.published: + self.store.outbox_published = True + self.store.current_transaction = None + return False + + +class FakeConnection: + def __init__(self, store): + self.store = store + + def transaction(self): + return FakeTransaction(self.store) + + def execute(self, query, params): + assert "UPDATE attachment_outbox" in query + assert params == ("outbox-row",) + self.store.current_transaction.published = True + + +class FakePool: + def __init__(self, store): + self.store = store + + def connection(self): + connection = FakeConnection(self.store) + + class ConnectionContext: + def __enter__(self_inner): + return connection + + def __exit__(self_inner, exc_type, exc_value, traceback): + return False + + return ConnectionContext() + + +class FakeOutboxStore: + def __init__(self): + self.pool = FakePool(self) + self.outbox_published = False + self.current_transaction = None + + def claim_outbox(self, connection): + if self.outbox_published: + return None + return { + "id": "outbox-row", + "job_id": JOB_ID, + "attachment_id": ATTACHMENT_ID, + "pipeline_version": 1, + "attempt": 1, + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + +class AmbiguousConfirmChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + self.messages.append(message) + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm") + + +class FailBeforePublishChannel: + def __init__(self): + self.messages = [] + self.fail_once = True + + def basic_publish(self, **message): + if self.fail_once: + self.fail_once = False + raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish") + self.messages.append(message) + + +def test_committed_outbox_survives_publisher_crash(): + store = FakeOutboxStore() + channel = FailBeforePublishChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher crash was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert channel.messages == [] + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 1 + return { + "scenario": "publisher_crash_before_publish", + "published_after_restart": len(channel.messages), + "outbox_recovered_after_restart": store.outbox_published, + } + + +def test_outbox_confirm_gap_republishes_same_job(): + store = FakeOutboxStore() + channel = AmbiguousConfirmChannel() + try: + publish_pending(store, channel) + raise AssertionError("publisher confirm fault was not injected") + except pika.exceptions.AMQPConnectionError: + pass + assert not store.outbox_published + assert publish_pending(store, channel) + assert store.outbox_published + assert len(channel.messages) == 2 + first, second = channel.messages + assert first["body"] == second["body"] + assert first["properties"].message_id == second["properties"].message_id == "outbox-row" + return { + "scenario": "publish_confirm_lost_after_broker_accept", + "deliveries": len(channel.messages), + "same_message_id": True, + } + + +def test_image_processing_has_a_hard_timeout(): + started = time.monotonic() + try: + _build_variants_bounded( + ATTACHMENT_ID, + b"bounded timeout test", + 1, + timeout_seconds=1, + builder=slow_builder, + ) + raise AssertionError("slow transform exceeded its time budget without termination") + except ProcessingTimeout: + pass + elapsed = time.monotonic() - started + assert elapsed < 4 + return { + "scenario": "transform_timeout_terminated", + "configured_timeout_seconds": 1, + "observed_seconds": round(elapsed, 3), + } + + +def main(): + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + results = [ + test_duplicate_and_ack_loss(content), + test_crash_before_and_after_side_effect(content, "after_claim"), + test_crash_before_and_after_side_effect(content, "after_original_put"), + test_reordered_attempts(content), + test_stale_pipeline_cannot_publish_newer_result(content), + test_committed_outbox_survives_publisher_crash(), + test_outbox_confirm_gap_republishes_same_job(), + test_image_processing_has_a_hard_timeout(), + ] + report = { + "result": "PASS", + "track": "2", + "fixture": "tests/fixtures/sample.png", + "scenarios": results, + "invariants": { + "confirmed_work_eventually_ready": True, + "duplicate_effects_create_no_second_variant_set": True, + "stale_attempt_writes_no_effects": True, + "stale_pipeline_cannot_publish_current_metadata": True, + "ambiguous_confirm_republishes_stable_job": True, + "delivery_semantics": "at-least-once; effects are fenced/idempotent", + }, + } + evidence = ROOT / "evidence" / "pipeline-faults.json" + evidence.parent.mkdir(exist_ok=True) + evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, sort_keys=True)) + print(f"Evidence written to {evidence.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_pipeline_replay_integration.py b/student-tests/test_pipeline_replay_integration.py new file mode 100644 index 0000000..8687a95 --- /dev/null +++ b/student-tests/test_pipeline_replay_integration.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Verify explicit replay changes the deployed pipeline while preserving the original.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +MARK_FAILED_CODE = """ +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + updated = conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', " + "variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'", + (sys.argv[1],), + ) + if updated.rowcount != 1: + raise RuntimeError("test attachment was not ready for failed replay injection") +finally: + store.close() +""" + +AUDIT_CODE = """ +import json +import sys +from src.store import Store + +store = Store() +store.open() +try: + with store.pool.connection() as conn: + rows = conn.execute( + "SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events " + "WHERE attachment_id = %s ORDER BY id", + (sys.argv[1],), + ).fetchall() + print(json.dumps(rows, sort_keys=True)) +finally: + store.close() +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def replay(container_id, attachment_id, version): + result = docker_exec( + container_id, + "python3", + "-m", + "src.replay_attachment", + "--attachment-id", + attachment_id, + "--pipeline-version", + str(version), + ) + return json.loads(result.stdout) + + +def main(): + args = arguments("Verify versioned attachment replay against the running Compose stack") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready_v1 = client.wait_ready(alice, attachment) + v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail") + v1_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch") + + containers = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.split() + check(bool(containers), "no running API container found") + replay_v2 = replay(containers[0], attachment["id"], 2) + check(replay_v2["status"] == "queued", "replay was not queued") + check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2") + check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id") + + ready_v2 = client.wait_ready(alice, ready_v1) + v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail") + v2_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + actor=alice, + )["bytes"] + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=alice, + )["bytes"] + check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding") + check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch") + check(original_bytes == original, "pipeline replay changed the original file") + + docker_exec( + containers[0], + "python3", + "-c", + MARK_FAILED_CODE, + attachment["id"], + ) + failed_replay = replay(containers[0], attachment["id"], 2) + check(failed_replay["status"] == "queued", "failed attachment replay was not queued") + check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version") + check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id") + ready_replayed = client.wait_ready(alice, ready_v2) + audit = json.loads( + docker_exec( + containers[0], + "python3", + "-c", + AUDIT_CODE, + attachment["id"], + ).stdout + ) + check( + [(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit] + == [(1, 2, "replay"), (2, 2, "replay")], + "pipeline replay audit trail is incomplete", + ) + check(ready_replayed["status"] == "ready", "failed attachment did not recover") + + print( + json.dumps( + { + "result": "PASS", + "previous_job_id": replay_v2["previous_job_id"], + "job_id": replay_v2["job_id"], + "pipeline_version": replay_v2["pipeline_version"], + "failed_replay_job_id": failed_replay["job_id"], + "audited_replays": len(audit), + "v1_thumbnail_sha256": v1_thumbnail["sha256"], + "v2_thumbnail_sha256": v2_thumbnail["sha256"], + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 0000000..ff12312 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Deterministic tests of the reconciliation decision rules without deleting real objects.""" +import os +import hashlib +import sys +import tempfile +from pathlib import Path + +os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret") +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from datetime import datetime, timedelta, timezone + +from src.reconcile import reconcile + + +class FakeObjects: + def __init__(self, items): + self.items = items + self.deleted = [] + + def objects(self): + return iter(self.items) + + def delete(self, key): + self.deleted.append(key) + self.items = [(name, date) for name, date in self.items if name != key] + + def put(self, key, content, content_type): + self.items.append((key, datetime.now(timezone.utc))) + + +class FakeStore: + def __init__(self): + now = datetime.now(timezone.utc) + self.metadata = { + "original/present": {"id": "present-id", "object_key": "original/present"}, + "original/missing": { + "id": "missing-id", + "object_key": "original/missing", + "size_bytes": 4, + "sha256": hashlib.sha256(b"test").hexdigest(), + "content_type": "image/png", + }, + } + self.objects = FakeObjects( + [ + ("original/present", now), + ("original/stale-orphan", now - timedelta(hours=2)), + ("original/new-orphan", now - timedelta(minutes=10)), + ] + ) + self.missing_incidents = set() + self.cleared_incidents = set() + + def reconciliation_snapshot(self): + return self.metadata + + def record_missing_object(self, attachment_id): + self.missing_incidents.add(attachment_id) + + def clear_missing_object(self, attachment_id): + self.missing_incidents.discard(attachment_id) + self.cleared_incidents.add(attachment_id) + + +def main(): + store = FakeStore() + dry_run = reconcile(store, grace_seconds=3600, apply_changes=False) + assert dry_run["stale_orphan_candidates"] == 1 + assert dry_run["missing_objects"] == 1 + assert dry_run["deleted_orphans"] == 0 + assert store.objects.deleted == [] and store.missing_incidents == set() + + applied = reconcile(store, grace_seconds=3600, apply_changes=True) + assert store.objects.deleted == ["original/stale-orphan"] + assert "original/new-orphan" not in store.objects.deleted + assert store.missing_incidents == {"missing-id"} + assert store.cleared_incidents == {"present-id"} + assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1 + + with tempfile.TemporaryDirectory() as backup: + restore_dir = Path(backup) + (restore_dir / "missing-id").write_bytes(b"test") + preview = reconcile(store, 3600, False, restore_dir) + assert preview["restorable_missing_objects"] == 1 + assert store.missing_incidents == {"missing-id"} + restored = reconcile(store, 3600, True, restore_dir) + assert restored["restored_missing_objects"] == 1 + assert store.missing_incidents == set() + + repeated = reconcile(store, grace_seconds=3600, apply_changes=True) + assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0 + assert store.objects.deleted == ["original/stale-orphan"] + assert store.missing_incidents == set() + print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_reconcile_integration.py b/student-tests/test_reconcile_integration.py new file mode 100644 index 0000000..78a719d --- /dev/null +++ b/student-tests/test_reconcile_integration.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects.""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +STORE_CODE = """ +import sys +from pathlib import Path +from src.store import Store + +store = Store() +store.open() +try: + action = sys.argv[1] + if action == "key": + attachment_id = sys.argv[2] + matches = [ + key for key, row in store.reconciliation_snapshot().items() + if str(row["id"]) == attachment_id and row["reconcile_type"] == "original" + ] + if len(matches) != 1: + raise RuntimeError("attachment metadata lookup failed") + print(matches[0]) + elif action == "put": + key, file_path = sys.argv[2:4] + content = Path(file_path).read_bytes() + store.objects.put(key, content, "image/png", len(content)) + elif action == "delete": + store.objects.delete(sys.argv[2]) + elif action == "exists": + print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no") + else: + raise RuntimeError("unsupported test operation") +finally: + store.close() +""" + +PRIVATE_CHECK_CODE = """ +import socket +import sys +import urllib.error +import urllib.request + +try: + socket.getaddrinfo("s3-core", 8888) +except socket.gaierror: + print("PASS: SeaweedFS master/filer network is isolated") +else: + raise RuntimeError("API network can resolve the SeaweedFS core service") + +for port in (8181, 9101): + try: + socket.create_connection(("s3", port), timeout=1).close() + except OSError: + continue + raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port)) + +try: + urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3) +except urllib.error.HTTPError as error: + if error.code == 403: + print("PASS: unauthenticated bucket access denied") + else: + raise RuntimeError("unexpected anonymous S3 status: " + str(error.code)) +else: + raise RuntimeError("private bucket allowed an unauthenticated GET") +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def reconciliation(container_id, *args): + result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args) + return json.loads(result.stdout) + + +def object_exists(container_id, key): + return docker_exec( + container_id, "python3", "-c", STORE_CODE, "exists", key + ).stdout.strip() == "yes" + + +def main(): + args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + containers = result.stdout.split() + check(len(containers) >= 2, "at least two running API replicas are required") + container_id = containers[0] + key = docker_exec( + container_id, "python3", "-c", STORE_CODE, "key", attachment["id"] + ).stdout.strip() + orphan_key = "original/" + str(uuid.uuid4()) + + with tempfile.TemporaryDirectory() as backup: + source = Path(backup) / "original.png" + source.write_bytes(content) + container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png" + recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex + recovery_file = recovery_dir + "/" + attachment["id"] + orphan_created = False + orphan_removed = False + missing_deleted = False + missing_restored = False + try: + subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True) + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file) + orphan_created = True + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key) + missing_deleted = True + + preview = reconciliation(container_id) + check( + preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1, + "dry-run missed injected inconsistencies", + ) + check(preview["deleted_orphans"] == 0, "dry-run modified S3") + immediate_apply = reconciliation(container_id, "--apply") + check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period") + + docker_exec(container_id, "mkdir", "-p", recovery_dir) + subprocess.run( + ["docker", "cp", str(source), container_id + ":" + recovery_file], + check=True, + ) + restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir) + check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable") + restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored") + missing_restored = True + + repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object") + check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period") + print("Waiting for the 60-second orphan grace period.") + time.sleep(61) + + cleanup_preview = reconciliation(container_id) + check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace") + cleaned = reconciliation(container_id, "--apply") + orphan_removed = not object_exists(container_id, orphan_key) + check(orphan_removed, "injected stale orphan was not removed") + final_scan = reconciliation(container_id, "--apply") + check( + final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0, + "repeat scan is not clean", + ) + finally: + if missing_deleted and not missing_restored: + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file) + if orphan_created and not orphan_removed: + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key) + + private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip() + check("PASS:" in private, "unauthenticated S3 access check failed") + downloaded = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(downloaded == content, "reconciler changed the original of an attachment used by a message") + check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference") + print( + json.dumps( + { + "result": "PASS", + "dataset_objects_scanned": final_scan["objects_seen"], + "metadata_records_scanned": final_scan["metadata_seen"], + "scan_duration_ms": final_scan["duration_ms"], + "orphan_detected_and_removed": cleaned["deleted_orphans"], + "missing_detected_and_restored": restored["restored_missing_objects"], + "fresh_orphan_preserved_until_grace": True, + "message_attachment_download_checksum": "unchanged", + "private_bucket": "anonymous request denied", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker.py b/student-tests/test_worker.py new file mode 100644 index 0000000..8e922fa --- /dev/null +++ b/student-tests/test_worker.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Local transformation checks for the lab5 image worker.""" +import hashlib +import io +import os +from pathlib import Path +import sys + +os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret") + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.worker import build_variants + + +def main(): + original = (ROOT / "tests/fixtures/sample.png").read_bytes() + attachment_id = "00000000-0000-4000-8000-000000000001" + variants = build_variants(attachment_id, original, pipeline_version=1) + by_name = {item["name"]: item for item in variants} + assert set(by_name) == {"thumbnail", "crop", "optimized"} + assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160) + assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128) + assert by_name["optimized"]["content_type"] == "image/webp" + for item in variants: + content = item["content"] + assert len(content) == item["size_bytes"] + assert hashlib.sha256(content).hexdigest() == item["sha256"] + with Image.open(io.BytesIO(content)) as image: + assert image.size == (item["width"], item["height"]) + newer = build_variants(attachment_id, original, pipeline_version=2) + assert all(f"/v2/" in item["object_key"] for item in newer) + assert all(f"/v1/" in item["object_key"] for item in variants) + assert by_name["thumbnail"]["sha256"] != { + item["name"]: item for item in newer + }["thumbnail"]["sha256"] + print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_worker_recovery_integration.py b/student-tests/test_worker_recovery_integration.py new file mode 100644 index 0000000..e2cb5fe --- /dev/null +++ b/student-tests/test_worker_recovery_integration.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Verify queued uploads and message links survive an API/worker restart.""" +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args): + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main(): + args = arguments("Verify queued attachment recovery after API and worker restart") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + compose("stop", "worker") + worker_stopped = True + try: + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not accepted while worker was stopped") + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message") + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + 409, + actor=bob, + ) + compose("restart", "api") + compose("start", "worker") + worker_stopped = False + client.call("GET", "/health/ready", schema="Health") + ready = client.wait_ready(bob, attachment) + original_bytes = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(original_bytes == original, "recovered worker changed original bytes") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages", + actor=bob, + schema="MessagePage", + )["json"] + recovered_message = next(item for item in page["items"] if item["id"] == message["id"]) + check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link") + print( + json.dumps( + { + "result": "PASS", + "initial_status": attachment["status"], + "final_status": ready["status"], + "message_attachment_link_survived": True, + "original_sha256": hashlib.sha256(original_bytes).hexdigest(), + }, + sort_keys=True, + ) + ) + finally: + if worker_stopped: + compose("start", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1_benchmark.py b/student-tests/track1_benchmark.py new file mode 100644 index 0000000..ed9fa38 --- /dev/null +++ b/student-tests/track1_benchmark.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +"""Repeat a paced synthetic message workload with telemetry off and sampled.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import platform +import re +import ssl +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + + +def command(args, capture=True): + result = subprocess.run( + args, + cwd=ROOT, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + return result.stdout.strip() if capture else "" + + +def set_env_value(text, name, value): + lines = text.splitlines() + prefix = name + "=" + for index, line in enumerate(lines): + if line.startswith(prefix): + lines[index] = prefix + value + break + else: + lines.append(prefix + value) + return "\n".join(lines) + "\n" + + +def parse_env(text): + values = {} + for line in text.splitlines(): + if line and not line.startswith("#") and "=" in line: + name, value = line.split("=", 1) + values[name] = value + return values + + +def wait_ready(url, ca_file=None, timeout=90): + deadline = time.monotonic() + timeout + context = ssl.create_default_context(cafile=ca_file) + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url + "/health/ready", context=context, timeout=2) as response: + if response.status == 200: + return + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(2) + raise RuntimeError("API did not become ready after the sampling mode change") + + +def parse_size(value): + match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value) + if not match: + return None + factors = { + "B": 1, + "kB": 1000, + "MB": 1000**2, + "GB": 1000**3, + "KiB": 1024, + "MiB": 1024**2, + "GiB": 1024**3, + } + return int(float(match.group(1)) * factors[match.group(2)]) + + +def docker_resources(): + rows = command( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}};{{.CPUPerc}};{{.MemUsage}}", + ] + ) + cpu_total = 0.0 + memory_total = 0 + found = False + for row in rows.splitlines(): + fields = row.split(";") + if len(fields) != 3 or not any( + name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-") + ): + continue + cpu = re.match(r"\s*([\d.]+)%", fields[1]) + memory = parse_size(fields[2].split("/", 1)[0]) + if cpu and memory is not None: + cpu_total += float(cpu.group(1)) + memory_total += memory + found = True + return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None + + +def storage_sizes(compose): + paths = { + "prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]), + "loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]), + "tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]), + } + result = {} + for name, args in paths.items(): + output = command(compose + args) + result[name] = int(output.split()[0]) + return result + + +def tempo_message_traces(compose, start, end): + url = ( + "http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000" + "&tags=service.name%%3Dmessenger-api" % (start, end) + ) + output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url]) + data = json.loads(output) + return [ + item + for item in data.get("traces", []) + if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages" + ] + + +def mean_spans(compose, traces, sample_limit=20): + counts = [] + for item in traces[:sample_limit]: + output = command( + compose + + [ + "exec", + "-T", + "tempo", + "wget", + "-qO-", + "http://127.0.0.1:3200/api/traces/" + item["traceID"], + ] + ) + data = json.loads(output) + count = sum( + len(scope.get("spans", [])) + for batch in data.get("batches", []) + for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", [])) + ) + counts.append(count) + return round(sum(counts) / len(counts), 2) if counts else 0.0 + + +def run_load(base_url, ca_file, duration, concurrency, target_rps, alice, chat): + from client import Client, Failure + + barrier = threading.Barrier(concurrency + 1) + ticket_lock = threading.Lock() + ticket = 0 + latencies = [] + failures = 0 + results_lock = threading.Lock() + start_clock = time.monotonic() + start_epoch = int(time.time()) + deadline = start_clock + duration + + def worker(worker_id): + nonlocal ticket, failures + client = Client(base_url, ca_file, timeout=5) + local_latencies = [] + local_failures = 0 + barrier.wait() + while True: + with ticket_lock: + current = ticket + ticket += 1 + due = start_clock + current / target_rps + if due >= deadline: + break + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + started = time.monotonic() + try: + client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current)) + local_latencies.append(time.monotonic() - started) + except Failure: + local_failures += 1 + with results_lock: + latencies.extend(local_latencies) + failures += local_failures + + samples = [] + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)] + barrier.wait() + while any(not future.done() for future in futures): + sample = docker_resources() + if sample is not None: + samples.append(sample) + time.sleep(2) + for future in futures: + future.result() + elapsed = time.monotonic() - start_clock + sorted_latencies = sorted(latencies) + + def percentile(value): + if not sorted_latencies: + return None + index = max(0, math.ceil(value * len(sorted_latencies)) - 1) + return round(sorted_latencies[index] * 1000, 3) + + total = len(sorted_latencies) + failures + report = { + "workload": "paced synthetic POST messages; no automatic retries", + "requested_duration_seconds": duration, + "target_rps": target_rps, + "elapsed_seconds": round(elapsed, 3), + "concurrency": concurrency, + "successful_requests": len(sorted_latencies), + "failed_requests": failures, + "error_ratio": round(failures / total, 6) if total else None, + "successful_rps": round(len(sorted_latencies) / elapsed, 3), + "successful_latency_ms": { + "p50": percentile(0.50), + "p95": percentile(0.95), + "p99": percentile(0.99), + }, + "docker_resource_samples": len(samples), + } + if samples: + report["docker_resources"] = { + "mean_cpu_percent_sum": round( + sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2 + ), + "peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples), + "peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples), + "services": ["api", "worker", "publisher", "otel-collector"], + } + return report, start_epoch, int(time.time()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", default=".env") + parser.add_argument("--base-url", default=None) + parser.add_argument("--ca-file", default=None) + parser.add_argument("--duration", type=int, default=15) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--target-rps", type=int, default=20) + parser.add_argument("--warmup-seconds", type=int, default=5) + parser.add_argument("--output", default="evidence/track1-observability-cost.json") + args = parser.parse_args() + if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1: + parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive") + + env_path = ROOT / args.env_file + if not env_path.is_file(): + parser.error("create the local .env from .env.example and generate secrets first") + original = env_path.read_text(encoding="utf-8") + values = parse_env(original) + base_url = args.base_url or "https://localhost:" + values.get("APP_PORT", "8443") + compose = ["docker", "compose", "--env-file", str(env_path)] + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + results = [] + configurations = [ + ("off", "true", "1.0"), + ("sample-100-percent", "false", "1.0"), + ("sample-25-percent", "false", "0.25"), + ("sample-10-percent", "false", "0.1"), + ] + try: + for name, disabled, ratio in configurations: + changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled) + changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio") + changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio) + env_path.write_text(changed, encoding="utf-8") + command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"]) + wait_ready(base_url, args.ca_file) + time.sleep(args.warmup_seconds) + from client import Client + + alice, _, _, chat = Client(base_url, args.ca_file, timeout=10).bootstrap() + time.sleep(2) + before = storage_sizes(compose) + load, start, end = run_load( + base_url, args.ca_file, args.duration, args.concurrency, args.target_rps, alice, chat + ) + time.sleep(15) + after = storage_sizes(compose) + traces = tempo_message_traces(compose, start - 5, end + 20) + spans_per_trace = mean_spans(compose, traces) + results.append( + { + "configuration": name, + "sdk_disabled": disabled == "true", + "trace_sampling_ratio": float(ratio), + "load": load, + "window_unix_seconds": {"start": start, "end": end}, + "sampled_message_trace_count": len(traces), + "estimated_spans_per_message_trace": spans_per_trace, + "persistent_storage_bytes_delta": { + signal: after[signal] - before[signal] for signal in before + }, + } + ) + + baseline = next( + item["estimated_spans_per_message_trace"] + for item in results + if item["configuration"] == "sample-100-percent" + ) + for item in results: + expected_spans = item["load"]["successful_requests"] * baseline + observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"] + item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2) + + report = { + "experiment": "telemetry overhead under fixed-rate synthetic POST load", + "hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.", + "method": { + "duration_seconds": args.duration, + "concurrency": args.concurrency, + "target_rps": args.target_rps, + "warmup_seconds": args.warmup_seconds, + "telemetry_flush_wait_seconds": 15, + "hardware": { + "platform": platform.platform(), + "processor": platform.processor() or "unspecified", + "logical_cpus": os.cpu_count(), + }, + "sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.", + "storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.", + }, + "limitations": [ + "The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.", + "Database history grows between modes; all modes use identical duration and concurrency.", + "Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.", + "Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.", + "CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.", + ], + "results": results, + "balanced_configuration": "sample-25-percent", + } + output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + env_path.write_text(original, encoding="utf-8") + command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"]) + + +if __name__ == "__main__": + main() diff --git a/student-tests/upload_concurrency.py b/student-tests/upload_concurrency.py new file mode 100644 index 0000000..e45746c --- /dev/null +++ b/student-tests/upload_concurrency.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Run ten simultaneous near-6-MiB uploads and sample API container memory.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +import re +import subprocess +import threading +import time +import uuid +from io import BytesIO +from pathlib import Path +import sys + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} +USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/") + + +def usage_bytes(line): + match = USAGE_RE.search(line) + return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0 + + +def api_containers(): + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + ids = result.stdout.split() + check(len(ids) >= 2, "at least two running API containers are required") + return ids + + +def sample_once(ids): + output = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + values = [usage_bytes(line) for line in output] + check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage") + return max(values) + + +def main(): + args = arguments("Bounded-memory ten-client upload experiment") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + width = height = 1400 + content = BytesIO() + Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save( + content, format="PNG", compress_level=0 + ) + payload = content.getvalue() + check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit") + ids = api_containers() + baseline = sample_once(ids) + samples = [] + stats_process = subprocess.Popen( + ["docker", "stats", "--format", "{{.MemUsage}}", *ids], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def sampler(): + if stats_process.stdout is None: + return + for line in stats_process.stdout: + value = usage_bytes(line) + if value: + samples.append(value) + + sampling_thread = threading.Thread(target=sampler, daemon=True) + sampling_thread.start() + count = 10 + barrier = threading.Barrier(count) + + def upload(index): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=15) + return worker.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png", + 202, + actor=alice, + raw=payload, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + started = time.monotonic() + try: + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(upload, range(count))) + finally: + stopped_by_test = False + if stats_process.poll() is None: + stats_process.kill() + stopped_by_test = True + try: + stats_process.wait(timeout=5) + except subprocess.TimeoutExpired: + stats_process.kill() + stats_process.wait(timeout=5) + sampling_thread.join(timeout=5) + stats_error = stats_process.stderr.read() if stats_process.stderr is not None else "" + check( + stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9), + "docker stats sampling process failed (exit " + + str(stats_process.returncode) + + "): " + + stats_error.strip()[:300], + ) + check(bool(samples), "docker stats produced no memory samples") + peak = max([baseline] + samples) + check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed") + print( + json.dumps( + { + "uploads": count, + "concurrency": count, + "bytes_per_upload": len(payload), + "duration_seconds": round(time.monotonic() - started, 3), + "api_containers": len(ids), + "sample_count": len(samples) // len(ids), + "baseline_max_container_bytes": baseline, + "peak_max_container_bytes": peak, + "peak_delta_bytes": max(0, peak - baseline), + "sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence", + "memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/README.md b/tests/README.md index 0dfa3f8..a83f73d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,12 +1,21 @@ -# Публичные проверки +# Проверки lab7 -Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. +HTTP проверки выполняются против запущенного Compose стенда; они не создают его автоматически. Клиенты Python используют стандартную библиотеку; сервер и unit tests требуют runtime зависимостей из `requirements.txt`. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. ```sh make check -make test BASE_URL=http://localhost:8080 -# без make: -python3 tests/smoke.py --base-url http://localhost:8080 +make test BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-security BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-primitives +make test-internal-mtls +make test-login-limit BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-openbao-acl PYTHON=.venv/bin/python +make test-worker-isolation +make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token +make test-leaf-rotation PYTHON=.venv/bin/python +make test-telemetry-redaction PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt" +make test-argon-budget +.venv/bin/pip-audit -r requirements.txt ``` `smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. @@ -15,10 +24,9 @@ python3 tests/smoke.py --base-url http://localhost:8080 ## Проверка сохранности, lab2+ -Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные. +`scripts/restart.sh` перезапускает API и используемые stores, сохраняя volumes и тестовые данные. ```sh -chmod +x scripts/restart.sh make test-persistence ACTION_SCRIPT=scripts/restart.sh ``` @@ -26,10 +34,9 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh ## Проверка отказа, lab3+ -Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо. +`scripts/stop-one.sh` сопоставляет `TARGET_INSTANCE_ID` с hostname API container и останавливает только его. Скрипт оставляет реплику остановленной; восстановление выполняйте отдельно. ```sh -chmod +x scripts/stop-one.sh make test-failover ACTION_SCRIPT=scripts/stop-one.sh ``` @@ -50,8 +57,14 @@ make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt ``` -Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях. +Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. `make test-internal-mtls` проверяет внутренний API listener из контейнера API: без клиентского сертификата, с недоверенной CA и с недопустимой URI SAN соединение отвергается, а LB identity проходит. `make test-login-limit` проверяет ответ 429 и Retry-After после серии неуспешных входов. Клиентский сертификат внешнему REST-клиенту не требуется: mTLS находится на связи LB→API. -## Что ещё остаётся доказать +`make test-openbao-acl` проверяет все сервисные AppRole: каждая identity читает собственный KV path и получает 403 на paths остальных identities; также проверяется ограниченный TTL токена. `make test-argon-budget` измеряет Argon2id verify latency, burst capacity и число одновременно занятых hash slots внутри API container с его CPU/RAM limits. Эти результаты характеризуют password hashing guard, а не end-to-end capacity стенда. `pip-audit -r requirements.txt` сканирует закреплённые Python runtime-зависимости по advisory database. -Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. +`make test-worker-isolation` запускается внутри worker container: проверяет ограниченную PostgreSQL-роль, запрет чтения/перечисления защищённых S3 buckets, разрешённые staging/originals/variants операции и запрет публикации worker в RabbitMQ. Временные probe objects удаляются после теста. + +`make test-worker-rotation ROOT_TOKEN_FILE=...` требует запущенный и распечатанный OpenBao и локальный путь к operator root-token file вне Git. Скрипт создаёт attachment при остановленном worker, выпускает новый SecretID, отзывает прежний, доказывает отказ старого credential, пересоздаёт только worker и проверяет `queued → ready`, checksum original и message link. Root token и SecretID не печатаются. Запуск создаёт синтетические записи и заменяет локальный worker SecretID. + +`make test-leaf-rotation` выпускает и устанавливает новый LB client leaf с прежней URI SAN, пересоздаёт API и HAProxy без build, проверяет отказ старого сертификата, приём нового и неизменность Docker image IDs. `make test-telemetry-redaction` генерирует синтетические credentials и сообщение, затем сканирует API log entries в Loki и трассы Tempo на отсутствие этих значений. + +Smoke и security-проверки рассчитаны на localhost Compose стенд, не на production. Нагрузочный benchmark, persistence hook и failover API выполняются отдельными командами выше; их машинные результаты записываются в `evidence/`. diff --git a/tests/argon2_budget.py b/tests/argon2_budget.py new file mode 100644 index 0000000..92b23dd --- /dev/null +++ b/tests/argon2_budget.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +import argparse +import json +import math +import resource +import statistics +from pathlib import Path +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src import security + + +def percentile(values, fraction): + ordered = sorted(values) + return ordered[max(0, math.ceil(len(ordered) * fraction) - 1)] + + +def main(): + parser = argparse.ArgumentParser(description="Measure Argon2id latency and bounded concurrency.") + parser.add_argument("--samples", type=int, default=20) + parser.add_argument("--concurrency", type=int, default=12) + args = parser.parse_args() + if args.samples < 1 or args.concurrency < 1: + parser.error("samples and concurrency must be positive") + + encoded = security.hash_password("argon2-budget-synthetic-password") + durations = [] + for _ in range(args.samples): + started = time.perf_counter() + if not security.verify_password("argon2-budget-synthetic-password", encoded): + raise RuntimeError("Argon2id self-verification failed") + durations.append(time.perf_counter() - started) + + active = 0 + peak_active = 0 + lock = threading.Lock() + original_slots = security._PASSWORD_HASH_SLOTS + + class TrackingSlots: + def acquire(self, timeout=None): + nonlocal active, peak_active + acquired = original_slots.acquire(timeout=timeout) + if acquired: + with lock: + active += 1 + peak_active = max(peak_active, active) + return acquired + + def release(self): + nonlocal active + with lock: + active -= 1 + original_slots.release() + + security._PASSWORD_HASH_SLOTS = TrackingSlots() + + def verify_once(_): + try: + return security.verify_password("argon2-budget-synthetic-password", encoded) + except security.PasswordHashCapacityError: + return False + + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + burst_results = list(executor.map(verify_once, range(args.concurrency))) + + print(json.dumps({ + "argon2_memory_kib": security._ARGON2_MEMORY_KIB, + "argon2_iterations": security._ARGON2_ITERATIONS, + "argon2_parallelism": security._ARGON2_PARALLELISM, + "hash_slots_per_process": security._PASSWORD_HASH_CONCURRENCY, + "hash_wait_seconds": security._PASSWORD_HASH_WAIT_SECONDS, + "sequential_samples": args.samples, + "verify_latency_ms": { + "p50": round(statistics.median(durations) * 1000, 3), + "p95": round(percentile(durations, 0.95) * 1000, 3), + "max": round(max(durations) * 1000, 3), + }, + "burst_requests": args.concurrency, + "burst_successes": sum(burst_results), + "burst_capacity_rejections": args.concurrency - sum(burst_results), + "peak_active_hashes": peak_active, + "max_rss_platform_units": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + "platform": sys.platform, + }, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/internal_mtls.py b/tests/internal_mtls.py new file mode 100644 index 0000000..4ca0a97 --- /dev/null +++ b/tests/internal_mtls.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify API listener client-certificate requirements from inside an API container.""" +import ssl +import http.client +import urllib.error +import urllib.request + + +HOST = "https://127.0.0.1:8080/health/ready" +CA = "/run/secrets/ca_crt" + + +def context(client_cert=None, client_key=None): + result = ssl.create_default_context(cafile=CA) + if client_cert: + result.load_cert_chain(client_cert, client_key) + return result + + +def rejected(client_cert=None, client_key=None): + try: + with urllib.request.urlopen(HOST, context=context(client_cert, client_key), timeout=5) as response: + response.read(1024) + except (urllib.error.URLError, ssl.SSLError, http.client.HTTPException, OSError): + return + raise AssertionError("API accepted a request with an unauthorized client identity") + + +def main(): + rejected() + rejected("/run/secrets/wrong_client_cert", "/run/secrets/wrong_client_key") + rejected("/run/secrets/foreign_client_cert", "/run/secrets/foreign_client_key") + + with urllib.request.urlopen( + HOST, + context=context("/run/secrets/lb_tls_cert", "/run/secrets/lb_tls_key"), + timeout=5, + ) as response: + if response.status != 200: + raise AssertionError(f"authorized LB identity returned HTTP {response.status}") + print("PASS: missing, foreign-CA and non-LB identities rejected; LB identity accepted.") + + +if __name__ == "__main__": + main() diff --git a/tests/leaf_certificate_rotation.py b/tests/leaf_certificate_rotation.py new file mode 100644 index 0000000..cb8cb4c --- /dev/null +++ b/tests/leaf_certificate_rotation.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Rotate the LB client leaf, pin the replacement, and prove the prior leaf is rejected.""" +import hashlib +import json +import os +from pathlib import Path +import ssl +import subprocess +import tempfile +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +PKI_DIR = ROOT / "secrets" / "pki" +CA_CERT = PKI_DIR / "ca.crt" +CA_KEY = PKI_DIR / "ca.key" +CERT_PATH = PKI_DIR / "lb-client.crt" +KEY_PATH = PKI_DIR / "lb-client.key" +BUNDLE_PATH = PKI_DIR / "lb-client.pem" +API_PROBE = """ +import json, ssl, sys, tempfile, urllib.request, urllib.error +value = json.load(sys.stdin) +with tempfile.TemporaryDirectory() as directory: + cert = directory + "/client.crt" + key = directory + "/client.key" + open(cert, "w").write(value["cert"]) + open(key, "w").write(value["key"]) + context = ssl.create_default_context(cafile="/run/secrets/ca_crt") + context.load_cert_chain(cert, key) + try: + with urllib.request.urlopen("https://127.0.0.1:8080/health/ready", context=context, timeout=5) as response: + result = {"accepted": response.status == 200} + except (urllib.error.URLError, ssl.SSLError, OSError): + result = {"accepted": False} + print(json.dumps(result)) +""" + + +def run(command, *, input_text=None): + return subprocess.run( + command, + cwd=ROOT, + input=input_text, + text=True, + check=True, + capture_output=True, + ) + + +def compose(*args, input_text=None): + return run(["docker", "compose", *args], input_text=input_text) + + +def write_atomic(path: Path, content: bytes, mode: int) -> None: + temporary = path.with_suffix(path.suffix + ".rotation-tmp") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode) + with os.fdopen(descriptor, "wb") as output: + output.write(content) + os.chmod(temporary, mode) + os.replace(temporary, path) + + +def probe(cert: bytes, key: bytes) -> bool: + payload = json.dumps( + { + "cert": cert.decode("ascii"), + "key": key.decode("ascii"), + } + ) + result = compose("exec", "-T", "api", "python3", "-c", API_PROBE, input_text=payload) + return json.loads(result.stdout)["accepted"] + + +def image_ids() -> set: + container_ids = compose("ps", "-q", "api").stdout.split() + if len(container_ids) != 2: + raise RuntimeError("expected exactly two running API replicas") + return { + run(["docker", "inspect", "--format", "{{.Image}}", container_id]).stdout.strip() + for container_id in container_ids + } + + +def wait_healthy(timeout: int = 90) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + containers = compose("ps", "-q", "api").stdout.split() + healthy = len(containers) == 2 + for container_id in containers: + state = json.loads( + run(["docker", "inspect", "--format", "{{json .State.Health}}", container_id]).stdout + ) + healthy = healthy and state.get("Status") == "healthy" + lb = compose("ps", "-q", "load_balancer").stdout.split() + healthy = healthy and len(lb) == 1 + if healthy: + state = json.loads( + run(["docker", "inspect", "--format", "{{json .State.Health}}", lb[0]]).stdout + ) + healthy = state.get("Status") == "healthy" + if healthy: + return + time.sleep(1) + raise RuntimeError("API and load balancer did not become healthy after certificate rotation") + + +def create_replacement(directory: Path) -> tuple[bytes, bytes, bytes]: + request_path = directory / "lb-client.csr" + cert_path = directory / "lb-client.crt" + key_path = directory / "lb-client.key" + extensions_path = directory / "lb-client.ext" + subprocess.run( + [ + "openssl", "req", "-new", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(key_path), "-out", str(request_path), + "-subj", "/CN=lab7-load-balancer", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + extensions_path.write_text( + "basicConstraints=critical,CA:FALSE\n" + "keyUsage=critical,digitalSignature,keyEncipherment\n" + "extendedKeyUsage=clientAuth\n" + "subjectAltName=URI:spiffe://mtusi2026lab7/load-balancer\n", + encoding="ascii", + ) + subprocess.run( + [ + "openssl", "x509", "-req", "-in", str(request_path), + "-CA", str(CA_CERT), "-CAkey", str(CA_KEY), "-CAcreateserial", + "-out", str(cert_path), "-days", "30", "-sha256", + "-extfile", str(extensions_path), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + cert = cert_path.read_bytes() + key = key_path.read_bytes() + return cert, key, cert + key + + +def main() -> None: + if not CA_CERT.is_file() or not CA_KEY.is_file(): + raise RuntimeError("local CA certificate and operator-held private key are required") + old_cert = CERT_PATH.read_bytes() + old_key = KEY_PATH.read_bytes() + old_bundle = BUNDLE_PATH.read_bytes() + before_images = image_ids() + if not probe(old_cert, old_key): + raise RuntimeError("current LB certificate was not accepted before rotation") + + replaced = False + try: + with tempfile.TemporaryDirectory(prefix="lab7-lb-leaf-") as directory_name: + new_cert, new_key, new_bundle = create_replacement(Path(directory_name)) + new_fingerprint = hashlib.sha256(ssl.PEM_cert_to_DER_cert(new_cert.decode("ascii"))).hexdigest() + write_atomic(CERT_PATH, new_cert, 0o644) + write_atomic(KEY_PATH, new_key, 0o600) + write_atomic(BUNDLE_PATH, new_bundle, 0o600) + replaced = True + + compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer") + wait_healthy() + if probe(old_cert, old_key): + raise RuntimeError("previous LB certificate remains authorized after rotation") + if not probe(new_cert, new_key): + raise RuntimeError("replacement LB certificate was not accepted") + + context = ssl.create_default_context(cafile=str(CA_CERT)) + with urllib.request.urlopen( + "https://localhost:8443/health/ready", + context=context, + timeout=10, + ) as response: + if response.status != 200: + raise RuntimeError("HTTPS ingress did not recover after leaf rotation") + + after_images = image_ids() + if before_images != after_images: + raise RuntimeError("container image changed; expected rotation without rebuild") + print( + json.dumps( + { + "result": "PASS", + "old_leaf_rejected": True, + "new_leaf_accepted": True, + "https_ingress_healthy": True, + "images_unchanged": True, + "new_leaf_sha256": new_fingerprint, + }, + sort_keys=True, + ) + ) + except Exception: + if replaced: + write_atomic(CERT_PATH, old_cert, 0o644) + write_atomic(KEY_PATH, old_key, 0o600) + write_atomic(BUNDLE_PATH, old_bundle, 0o600) + compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer") + raise + + +if __name__ == "__main__": + main() diff --git a/tests/login_rate_limit.py b/tests/login_rate_limit.py new file mode 100644 index 0000000..be671b0 --- /dev/null +++ b/tests/login_rate_limit.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import base64 +import uuid + +from client import Client, arguments, check, run + + +def main(): + args = arguments("Verify the shared login rate limit") + client = Client(args.base_url, args.ca_file) + username = "limit_" + uuid.uuid4().hex[:16] + client.call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": "Rate limit test", "password": "Correct-Password-123"}, + schema="User", + ) + invalid = base64.b64encode((username + ":Wrong-Password-123").encode()).decode("ascii") + for _ in range(5): + client.call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + invalid}, + ) + result = client.call( + "POST", + "/api/v1/auth/sessions", + 429, + headers={"Authorization": "Basic " + invalid}, + ) + check(result["json"]["error"]["code"] == "rate_limited", "expected rate_limited response") + check(int(result["headers"].get("Retry-After", "0")) > 0, "missing positive Retry-After") + print("PASS: sixth failed login in the window is rejected with 429 and Retry-After.") + + +if __name__ == "__main__": + run(main) diff --git a/tests/openbao_acl.py b/tests/openbao_acl.py new file mode 100644 index 0000000..a631ac0 --- /dev/null +++ b/tests/openbao_acl.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +import argparse +import json +import ssl +import urllib.error +import urllib.request +from pathlib import Path + + +SERVICES = ("api", "worker", "publisher", "migrate", "reconciler") + + +class BaoClient: + def __init__(self, address: str, ca_file: str): + self.address = address.rstrip("/") + self.context = ssl.create_default_context(cafile=ca_file) + + def request(self, method: str, path: str, payload=None, token: str = ""): + body = json.dumps(payload).encode("utf-8") if payload is not None else None + headers = {"Content-Type": "application/json"} + if token: + headers["X-Vault-Token"] = token + request = urllib.request.Request( + self.address + "/v1/" + path, + data=body, + headers=headers, + method=method, + ) + try: + with urllib.request.urlopen(request, context=self.context, timeout=5) as response: + return response.status, json.loads(response.read(1024 * 1024)) + except urllib.error.HTTPError as error: + return error.code, {} + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error: + raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check service AppRole path isolation.") + parser.add_argument("--address", default="https://localhost:8200") + parser.add_argument("--ca-file", required=True) + parser.add_argument("--credential-dir", default="secrets/openbao") + args = parser.parse_args() + credential_dir = Path(args.credential_dir) + client = BaoClient(args.address, args.ca_file) + + for service in SERVICES: + role_id = (credential_dir / f"{service}_role_id").read_text(encoding="utf-8").strip() + secret_id = (credential_dir / f"{service}_secret_id").read_text(encoding="utf-8").strip() + status, response = client.request( + "POST", + "auth/approle/login", + {"role_id": role_id, "secret_id": secret_id}, + ) + token = response.get("auth", {}).get("client_token") + if status != 200 or not token: + raise RuntimeError(f"{service} AppRole authentication failed with HTTP {status}") + lease_duration = response.get("auth", {}).get("lease_duration") + if not isinstance(lease_duration, int) or not 1 <= lease_duration <= 3600: + raise RuntimeError(f"{service} AppRole returned an invalid token TTL") + own_status, _ = client.request( + "GET", + f"secret/data/messenger/{service}", + token=token, + ) + if own_status != 200: + raise RuntimeError(f"{service} cannot read its own scoped KV path (HTTP {own_status})") + for other in SERVICES: + if other == service: + continue + denied_status, _ = client.request( + "GET", + f"secret/data/messenger/{other}", + token=token, + ) + if denied_status != 403: + raise RuntimeError( + f"{service} AppRole accessed {other} KV path (HTTP {denied_status})" + ) + lookup_status, lookup_response = client.request( + "GET", + "auth/token/lookup-self", + token=token, + ) + token_data = lookup_response.get("data", {}) + if lookup_status != 200 or not 1 <= token_data.get("ttl", 0) <= 3600: + raise RuntimeError( + f"{service} AppRole token TTL is not bounded (HTTP {lookup_status})" + ) + + print("PASS: each AppRole reads its own KV path and is denied all other service paths.") + + +if __name__ == "__main__": + main() diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700..e643f71 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -36,7 +36,7 @@ def main(): script = Path(args.action_script).expanduser().resolve() check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') check('.example.' not in script.name,'copy and implement the example hook first') - c = Client(args.base_url,args.ca_file,timeout=2) + c = Client(args.base_url,args.ca_file,timeout=15) alice,bob,eve,chat = c.bootstrap() sent = c.message(alice,chat,'Сохранить при отказе') original = sent['json'] diff --git a/tests/telemetry_redaction.py b/tests/telemetry_redaction.py new file mode 100644 index 0000000..57d2b8e --- /dev/null +++ b/tests/telemetry_redaction.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Verify synthetic credentials and message bodies are absent from Loki logs and Tempo spans.""" +import base64 +import json +from pathlib import Path +import re +import subprocess +import time +import uuid + +from client import Client, arguments, check, run + + +ROOT = Path(__file__).resolve().parents[1] +TELEMETRY_PROBE = r""" +import json, ssl, sys, time, urllib.parse, urllib.request +probe = json.load(sys.stdin) +context = ssl.create_default_context(cafile="/run/secrets/ca_crt") +def request(url): + with urllib.request.urlopen(url, context=context, timeout=8) as response: + return json.load(response) +log_query = '{service_name="messenger-api"}' +deadline = time.monotonic() + 30 +trace_ids = [] +log_entries = [] +while time.monotonic() < deadline: + params = urllib.parse.urlencode({"query": log_query, "start": probe["start_ns"], "end": str(time.time_ns()), "limit": "1000"}) + logs = request("https://loki:3100/loki/api/v1/query_range?" + params) + log_entries = [ + {"labels": stream.get("stream", {}), "line": line} + for stream in logs.get("data", {}).get("result", []) + for _, line in stream.get("values", []) + ] + search = urllib.parse.urlencode({"limit": "1000", "start": probe["start"], "end": str(int(time.time())), "tags": "service.name=messenger-api"}) + found = request("https://tempo:3200/api/search?" + search) + trace_ids = [item["traceID"] for item in found.get("traces", []) if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"] + if trace_ids and log_entries: + break + time.sleep(1) +if not trace_ids: + raise RuntimeError("no API message trace appeared in Tempo within the deadline") +payloads = [json.dumps(log_entries, ensure_ascii=False)] +for trace_id in trace_ids: + payloads.append(json.dumps(request("https://tempo:3200/api/traces/" + trace_id), ensure_ascii=False)) +leaked = [value for value in probe["values"] if any(value in payload for payload in payloads)] +print(json.dumps({"trace_count": len(trace_ids), "log_entries": len(log_entries), "leak_count": len(leaked)})) +""" + + +def main() -> None: + args = arguments("Verify request secrets and message bodies are excluded from telemetry") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + canary = "telemetry-canary-" + uuid.uuid4().hex + username = "u_probe_" + uuid.uuid4().hex[:12] + password = "Password-" + uuid.uuid4().hex + start_time = time.time() + user = client.call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": canary, "password": password}, + schema="User", + )["json"] + auth_value = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + session = client.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + auth_value}, + schema="Session", + )["json"] + check(session["user"]["id"] == user["id"], "telemetry probe user session did not authenticate") + client.call( + "GET", + "/api/v1/users/me", + 401, + headers={"Authorization": "Bearer " + canary}, + ) + client.message(alice, chat, canary) + values = [canary, username, password, auth_value, session["token"], alice["token"]] + probe = { + "values": values, + "start": str(max(0, int(start_time) - 5)), + "start_ns": str(max(0, int(start_time - 5) * 1_000_000_000)), + } + result = subprocess.run( + [ + "docker", "compose", "exec", "-T", "api", + "python3", "-c", TELEMETRY_PROBE, + ], + cwd=ROOT, + input=json.dumps(probe), + text=True, + capture_output=True, + check=True, + ) + telemetry = json.loads(result.stdout) + check(telemetry["trace_count"] > 0, "no API traces were scanned") + check(telemetry["leak_count"] == 0, "synthetic credentials or message content appeared in telemetry") + print( + json.dumps( + { + "result": "PASS", + "synthetic_values_checked": len(values), + "api_traces_scanned": telemetry["trace_count"], + "log_entries_scanned": telemetry["log_entries"], + "leaks": 0, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/test_security_primitives.py b/tests/test_security_primitives.py new file mode 100644 index 0000000..d7716f6 --- /dev/null +++ b/tests/test_security_primitives.py @@ -0,0 +1,54 @@ +import base64 +import hashlib +import unittest +from unittest.mock import patch + +from src.security import ( + PasswordHashCapacityError, + allowed_client_identity, + hash_password, + password_hash_needs_upgrade, + verify_password, +) + + +class PasswordHashTests(unittest.TestCase): + def test_argon2id_verifies_unicode_password(self): + encoded = hash_password("корректный-password-123") + self.assertTrue(encoded.startswith("$argon2id$")) + self.assertIn("m=19456,t=2,p=1", encoded) + self.assertTrue(verify_password("корректный-password-123", encoded)) + self.assertFalse(verify_password("другой-password-123", encoded)) + self.assertFalse(password_hash_needs_upgrade(encoded)) + oversized = "$argon2id$v=19$m=999999,t=2,p=1$c2FsdA$ZGlnaWVzdA" + self.assertFalse(verify_password("attacker-password-123", oversized)) + + def test_legacy_scrypt_hash_is_verified_and_marked_for_upgrade(self): + password = "legacy-password-123" + salt = b"lab6-test-salt!!" + digest = hashlib.scrypt( + password.encode(), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + stored = "scrypt$32768$8$1${}${}".format( + base64.urlsafe_b64encode(salt).decode(), + base64.urlsafe_b64encode(digest).decode(), + ) + self.assertTrue(verify_password(password, stored)) + self.assertTrue(password_hash_needs_upgrade(stored)) + + def test_only_lb_service_identity_is_authorized(self): + trusted = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/load-balancer")]} + other = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/other-service")]} + self.assertTrue(allowed_client_identity(trusted)) + self.assertFalse(allowed_client_identity(other)) + self.assertFalse(allowed_client_identity({})) + + def test_password_hash_capacity_fails_fast(self): + with patch("src.security._PASSWORD_HASH_SLOTS") as slots: + slots.acquire.return_value = False + with self.assertRaises(PasswordHashCapacityError): + verify_password("password", "$argon2id$malformed") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/worker_identity_rotation.py b/tests/worker_identity_rotation.py new file mode 100644 index 0000000..8f2aea3 --- /dev/null +++ b/tests/worker_identity_rotation.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Revoke the worker AppRole SecretID and recover a queued attachment with a replacement.""" +import json +import os +from pathlib import Path +import ssl +import subprocess +import urllib.error +import urllib.request + +from client import Client, arguments, check, run + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" / "openbao" + + +class BaoClient: + def __init__(self, ca_file: str, root_token_file: str): + self.address = "https://localhost:8200" + self.context = ssl.create_default_context(cafile=ca_file) + self.root_token = Path(root_token_file).read_text(encoding="utf-8").strip() + check(bool(self.root_token), "operator root-token file is empty") + + def request(self, method: str, path: str, payload=None, token=None): + body = json.dumps(payload).encode("utf-8") if payload is not None else None + headers = {"Content-Type": "application/json"} + if token is not None: + if token: + headers["X-Vault-Token"] = token + elif self.root_token: + headers["X-Vault-Token"] = self.root_token + request = urllib.request.Request( + self.address + "/v1/" + path, + data=body, + headers=headers, + method=method, + ) + try: + with urllib.request.urlopen(request, context=self.context, timeout=8) as response: + data = response.read(1024 * 1024) + return response.status, json.loads(data) if data else {} + except urllib.error.HTTPError as error: + return error.code, {} + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error: + raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None + + +def compose(*args) -> None: + subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def save_secret_id(value: str) -> None: + path = SECRET_DIR / "worker_secret_id" + temporary = path.with_suffix(".tmp") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + os.replace(temporary, path) + + +def rotate_worker_secret(bao: BaoClient) -> None: + role_id = (SECRET_DIR / "worker_role_id").read_text(encoding="utf-8").strip() + old_secret_id = (SECRET_DIR / "worker_secret_id").read_text(encoding="utf-8").strip() + check(bool(role_id) and bool(old_secret_id), "worker AppRole credential files are empty") + + status, created = bao.request( + "POST", + "auth/approle/role/worker/secret-id", + {}, + ) + new_secret_id = created.get("data", {}).get("secret_id") + check(status == 200 and isinstance(new_secret_id, str), "OpenBao did not issue a replacement SecretID") + save_secret_id(new_secret_id) + + status, login = bao.request( + "POST", + "auth/approle/login", + {"role_id": role_id, "secret_id": new_secret_id}, + token="", + ) + token = login.get("auth", {}).get("client_token") + check(status == 200 and isinstance(token, str), "replacement worker SecretID did not authenticate") + status, _ = bao.request( + "GET", + "secret/data/messenger/worker", + token=token, + ) + check(status == 200, "replacement worker identity cannot read its own secret path") + + status, _ = bao.request( + "POST", + "auth/approle/role/worker/secret-id/destroy", + {"secret_id": old_secret_id}, + ) + check(status == 204, "OpenBao did not revoke the previous worker SecretID") + status, _ = bao.request( + "POST", + "auth/approle/login", + {"role_id": role_id, "secret_id": old_secret_id}, + token="", + ) + check(status == 400, "revoked worker SecretID still authenticates") + + +def main() -> None: + def configure(parser): + parser.add_argument("--root-token-file", required=True) + + args = arguments("Verify queued attachment recovery after worker SecretID rotation", configure) + ca_file = args.ca_file or str(ROOT / "secrets" / "pki" / "ca.crt") + client = Client(args.base_url, ca_file) + bao = BaoClient(ca_file, args.root_token_file) + alice, bob, _, chat = client.bootstrap() + worker_stopped = False + try: + compose("stop", "worker") + worker_stopped = True + attachment, original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not queued while the worker was stopped") + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to its message") + + rotate_worker_secret(bao) + compose("up", "-d", "--no-deps", "--force-recreate", "worker") + worker_stopped = False + + ready = client.wait_ready(bob, attachment) + original_bytes = client.call( + "GET", + f"/api/v1/attachments/{attachment['id']}/content", + actor=bob, + )["bytes"] + check(original_bytes == original, "rotated worker changed the original attachment bytes") + page = client.call( + "GET", + f"/api/v1/chats/{chat['id']}/messages", + actor=bob, + schema="MessagePage", + )["json"] + recovered = next(item for item in page["items"] if item["id"] == message["id"]) + check(recovered["attachment_ids"] == [attachment["id"]], "message lost its attachment link") + print( + json.dumps( + { + "result": "PASS", + "old_worker_secret_id_revoked": True, + "replacement_worker_secret_id_authenticated": True, + "attachment_state": [attachment["status"], ready["status"]], + "original_sha256_preserved": True, + "message_attachment_link_preserved": True, + }, + sort_keys=True, + ) + ) + finally: + if worker_stopped: + compose("up", "-d", "--no-deps", "--force-recreate", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/tests/worker_isolation.py b/tests/worker_isolation.py new file mode 100644 index 0000000..bfde060 --- /dev/null +++ b/tests/worker_isolation.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Verify the worker identity is restricted to its required database, S3, and broker actions.""" +import json +from pathlib import Path +import sys +import uuid + +import pika +import psycopg +from botocore.exceptions import ClientError + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.jobs import connect +from src.object_store import ObjectStore +from src.store import database_url + + +def check_database() -> dict: + with psycopg.connect(database_url()) as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT current_user") + role = cursor.fetchone()[0] + cursor.execute("SELECT id, status FROM attachments LIMIT 1") + cursor.fetchall() + try: + cursor.execute("SELECT id FROM users LIMIT 1") + except psycopg.errors.InsufficientPrivilege: + connection.rollback() + users_denied = True + else: + raise AssertionError("worker can read users") + if role != "messenger_worker": + raise AssertionError(f"unexpected worker database role: {role}") + return {"role": role, "attachments_read": True, "users_read_denied": users_denied} + + +def check_s3() -> dict: + client = ObjectStore().client + results = {} + for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants"): + try: + client.list_objects_v2(Bucket=bucket, MaxKeys=1) + except ClientError as error: + code = error.response.get("Error", {}).get("Code") + if code not in ("AccessDenied", "403"): + raise + results[f"{bucket}_list_denied"] = True + else: + raise AssertionError(f"worker can list {bucket}") + + for bucket, key in ( + ("messenger-attachments", f"original/{uuid.uuid4()}"), + ("messenger-variants", f"variants/{uuid.uuid4()}"), + ): + try: + client.get_object(Bucket=bucket, Key=key) + except ClientError as error: + code = error.response.get("Error", {}).get("Code") + if code not in ("AccessDenied", "403"): + raise + results[f"{bucket}_read_denied"] = True + else: + raise AssertionError(f"worker can read objects from {bucket}") + + probe = str(uuid.uuid4()) + allowed_objects = ( + ("messenger-staging", f"staging/isolation-probe/{probe}", True), + ("messenger-attachments", f"original/isolation-probe/{probe}", False), + ("messenger-variants", f"variants/isolation-probe/{probe}", False), + ) + for bucket, key, verify_read in allowed_objects: + try: + client.put_object(Bucket=bucket, Key=key, Body=b"worker-scope-probe") + if verify_read: + response = client.get_object(Bucket=bucket, Key=key) + try: + if response["Body"].read() != b"worker-scope-probe": + raise AssertionError("worker staging read returned unexpected data") + finally: + response["Body"].close() + results[f"{bucket}_write"] = True + finally: + try: + client.delete_object(Bucket=bucket, Key=key) + except ClientError as error: + if error.response.get("Error", {}).get("Code") not in ("NoSuchKey", "404"): + raise + results["messenger-staging_read"] = True + results["temporary_objects_removed"] = True + return results + + +def check_broker() -> dict: + connection = connect() + channel = connection.channel() + try: + try: + channel.basic_publish( + exchange="", + routing_key="attachments.process.v1", + body=b"worker-isolation-probe", + ) + channel.close() + except pika.exceptions.ChannelClosedByBroker as error: + if error.reply_code != 403: + raise + publish_denied = True + else: + publish_denied = False + if not publish_denied: + raise AssertionError("worker can publish to the processing queue") + return {"broker_identity_connected": True, "publish_denied": publish_denied} + finally: + if channel.is_open: + channel.close() + if connection.is_open: + connection.close() + + +def main() -> None: + result = { + "database": check_database(), + "s3": check_s3(), + "broker": check_broker(), + } + print(json.dumps({"result": "PASS", **result}, sort_keys=True)) + + +if __name__ == "__main__": + main() -- 2.54.0