From baafe2a3aef2df177ca61c235077d85599a09e50 Mon Sep 17 00:00:00 2001 From: ViktoriyaKa Date: Sat, 7 Nov 2026 20:46:02 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=81=D1=82=D0=B5=D0=BD=D0=B4=20=D0=BB=D0=B0=D0=B1?= =?UTF-8?q?=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD=D0=BE=D0=B9=207=20?= =?UTF-8?q?=D1=81=20mTLS,=20OpenBao=20=D0=B8=20least=20privilege=20worker.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи. --- .github/pull_request_template.md | 28 +- .gitignore | 2 + Dockerfile | 22 + REPORT.md | 85 ++- app/__init__.py | 1 + app/asgi.py | 124 ++++ app/broker.py | 140 ++++ app/codec.py | 139 ++++ app/db.py | 98 +++ app/domain.py | 611 ++++++++++++++++++ app/entrypoint.py | 32 + app/errors.py | 49 ++ app/faults.py | 73 +++ app/http.py | 221 +++++++ app/images.py | 62 ++ app/main.py | 53 ++ app/mtls_proxy.py | 109 ++++ app/poller.py | 42 ++ app/probes/__init__.py | 0 app/probes/security.py | 331 ++++++++++ app/probes/track1.py | 125 ++++ app/ratelimit.py | 61 ++ app/schema.sql | 108 ++++ app/secrets_loader.py | 137 ++++ app/settings.py | 88 +++ app/storage.py | 70 ++ app/telemetry.py | 389 +++++++++++ app/worker.py | 348 ++++++++++ compose.yaml | 533 +++++++++++++++ contracts/trust-matrix.md | 48 +- docs/runbooks/api-error-budget.md | 18 + docs/runbooks/db-latency.md | 15 + docs/runbooks/queue-lag.md | 14 + docs/runbooks/telemetry-backend.md | 8 + docs/runbooks/time-to-ready.md | 11 + evidence/argon2-budget.json | 12 + evidence/track1-least-privilege.json | 12 + evidence/track1-run.json | 41 ++ evidence/verify-local.json | 27 + faults/README.md | 11 + infra/grafana/dashboards/messenger.json | 187 ++++++ .../provisioning/dashboards/dashboards.yml | 12 + .../provisioning/datasources/datasources.yml | 53 ++ infra/loki/loki.yml | 44 ++ infra/minio/api-s3.json | 24 + infra/minio/worker-broad.json | 10 + infra/minio/worker-s3.json | 24 + infra/nginx/ingress.conf | 34 + infra/nginx/nginx.conf | 15 + infra/nginx/otel-edge.conf | 26 + infra/openbao/config.hcl | 18 + infra/openbao/policies/api.hcl | 27 + infra/openbao/policies/worker.hcl | 27 + infra/otel/collector.yaml | 103 +++ infra/postgres/03-grants.sql | 13 + infra/postgres/Dockerfile | 7 + infra/postgres/entrypoint.sh | 16 + infra/postgres/pg_hba.conf | 3 + infra/prometheus/prometheus.yml | 16 + infra/prometheus/rules.yml | 96 +++ infra/prometheus/web.yml | 4 + infra/rabbitmq/Dockerfile | 7 + infra/rabbitmq/definitions.json | 33 + infra/rabbitmq/entrypoint.sh | 9 + infra/rabbitmq/rabbitmq.conf | 17 + infra/tempo/tempo.yml | 41 ++ infra/valkey/valkey.conf | 14 + requirements.txt | 11 + scripts/bootstrap.py | 544 ++++++++++++++++ scripts/restart.sh | 30 + scripts/stop-one.sh | 23 + scripts/track1_run.py | 312 +++++++++ scripts/unseal.py | 62 ++ scripts/verify_local.py | 325 ++++++++++ security/THREAT_MODEL.md | 36 +- security/scan-triage.md | 17 + 76 files changed, 6473 insertions(+), 65 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/asgi.py create mode 100644 app/broker.py create mode 100644 app/codec.py create mode 100644 app/db.py create mode 100644 app/domain.py create mode 100644 app/entrypoint.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/http.py create mode 100644 app/images.py create mode 100644 app/main.py create mode 100644 app/mtls_proxy.py create mode 100644 app/poller.py create mode 100644 app/probes/__init__.py create mode 100644 app/probes/security.py create mode 100644 app/probes/track1.py create mode 100644 app/ratelimit.py create mode 100644 app/schema.sql create mode 100644 app/secrets_loader.py create mode 100644 app/settings.py create mode 100644 app/storage.py create mode 100644 app/telemetry.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 docs/runbooks/api-error-budget.md create mode 100644 docs/runbooks/db-latency.md create mode 100644 docs/runbooks/queue-lag.md create mode 100644 docs/runbooks/telemetry-backend.md create mode 100644 docs/runbooks/time-to-ready.md create mode 100644 evidence/argon2-budget.json create mode 100644 evidence/track1-least-privilege.json create mode 100644 evidence/track1-run.json create mode 100644 evidence/verify-local.json create mode 100644 faults/README.md create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/grafana/provisioning/datasources/datasources.yml create mode 100644 infra/loki/loki.yml create mode 100644 infra/minio/api-s3.json create mode 100644 infra/minio/worker-broad.json create mode 100644 infra/minio/worker-s3.json create mode 100644 infra/nginx/ingress.conf create mode 100644 infra/nginx/nginx.conf create mode 100644 infra/nginx/otel-edge.conf create mode 100644 infra/openbao/config.hcl create mode 100644 infra/openbao/policies/api.hcl create mode 100644 infra/openbao/policies/worker.hcl create mode 100644 infra/otel/collector.yaml create mode 100644 infra/postgres/03-grants.sql create mode 100644 infra/postgres/Dockerfile create mode 100644 infra/postgres/entrypoint.sh create mode 100644 infra/postgres/pg_hba.conf create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules.yml create mode 100644 infra/prometheus/web.yml create mode 100644 infra/rabbitmq/Dockerfile create mode 100644 infra/rabbitmq/definitions.json create mode 100644 infra/rabbitmq/entrypoint.sh create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 infra/tempo/tempo.yml create mode 100644 infra/valkey/valkey.conf create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.sh create mode 100644 scripts/track1_run.py create mode 100644 scripts/unseal.py create mode 100644 scripts/verify_local.py create mode 100644 security/scan-triage.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..110d91d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 7 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Казанцева Виктория, БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 1, компрометация worker и least privilege -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: бизнес-API lab6 сохранён и вынесен за HTTPS. Добавлены Argon2id, общий rate limit, обязательный mTLS LB→API, TLS к PostgreSQL, SeaweedFS, RabbitMQ, Valkey, OpenBao и telemetry, scoped cert-auth в OpenBao и отдельные права API/worker. +Команды запуска из чистого clone: `python scripts/bootstrap.py`, затем `python scripts/bootstrap.py --up`. Проверки и наблюдения — в REPORT.md. +Результаты проверок и ссылка на REPORT.md: `check_contract` PASS, smoke PASS, `test-security` PASS, внутренний probe 29/29 PASS, track1 до/после least privilege PASS, queued-задача пережила отзыв и смену S3 secret, leaf LB сменён без пересборки, sealed OpenBao даёт 503. См. REPORT.md. +Ограничения / известные ошибки: на Windows курс не исполняет `.sh` напрямую; hook прогнаны через Git Bash. `docker scout` без входа в Docker Hub базу CVE образа не отдал; разбор пакетов — `security/scan-triage.md`. +Источники: OWASP Password Storage, OpenBao, RabbitMQ EXTERNAL, SeaweedFS, Docker security. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a7189..5136ac9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ data/ backups/ # Runtime/build artifacts .venv/ +.venv-bootstrap/ +run/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d85cad3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + HOME=/tmp \ + AWS_EC2_METADATA_DISABLED=true + +WORKDIR /app + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY app /app/app + +USER 10001:10001 +EXPOSE 8443 + +CMD ["python", "-m", "app.entrypoint"] diff --git a/REPORT.md b/REPORT.md index 9801db4..98e5b04 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,86 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Казанцева Виктория, БПИ2301 +- Целевая оценка; выбранный трек 5: 5, трек 1 (компрометация worker / least privilege) +- Commit/PR: локально, не публиковалось +- Источники: OWASP Password Storage, OpenBao AppRole/policies/hardening, Docker security, документация RabbitMQ EXTERNAL и SeaweedFS S3. Секреты, root token и unseal keys в отчёт не переносились. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop, Python 3.9+ на хосте и пакет `cryptography` (bootstrap ставит его в `.venv-bootstrap`, если импорта нет). На Windows публичные тесты запускаются через `python`, не `python3`. + +```text +python scripts/bootstrap.py +python scripts/bootstrap.py --up +``` + +Первая команда создаёт локальные CA, leaf-сертификаты и пароли в `secrets/` и `.env`. Вторая инициализирует OpenBao (Shamir 3/2, file storage, не dev-mode), если том ещё пустой, записывает scoped KV, собирает образы и поднимает стенд. Root token и unseal keys остаются в `secrets/operator/init.json` и печатаются только как «sealed=true/false» скриптом `python scripts/unseal.py`. + +Публичная точка: `https://127.0.0.1:8443`. CA: `secrets/pki/ca.crt`. Grafana: `https://127.0.0.1:3001` (порт 3000 на этой машине уже был занят другим стендом). OpenBao и management RabbitMQ наружу не опубликованы. + +Проверки: + +```text +python scripts/check_contract.py +python tests/smoke.py --base-url https://127.0.0.1:8443 --ca-file /secrets/pki/ca.crt +python tests/security.py --base-url https://127.0.0.1:8443 --ca-file /secrets/pki/ca.crt +docker compose --env-file .env --profile tools run --rm probe +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных томов: `docker compose --env-file .env down -v` (это стирает БД, S3, очередь и OpenBao; после этого нужен новый `--up`). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент говорит с nginx по HTTPS (`127.0.0.1:8443`). Nginx ходит к `api-1:8443` и `api-2:8443` с клиентским сертификатом LB. Процесс API принимает mTLS и пускает только SAN `lb.messenger.internal` плюс serial из allowlist, затем отдаёт HTTP на `127.0.0.1:8080`. Этот порт из сети `app` не открывается. + +Бизнес-API lab6 сохранён: пользователи, чаты, сообщения, идемпотентность, вложения `queued/processing/ready/failed`, thumbnail. Пароли — Argon2id, в хеше есть salt и параметры. Сессия — случайные 256 бит, в БД только SHA-256. Вход ограничен общим счётчиком Valkey до вычисления Argon2. + +Секреты БД, S3 и Valkey API читает из OpenBao по своему leaf-сертификату. Root token в контейнеры приложения не монтируется. S3 — SeaweedFS 4.51 под именем сервиса `minio`, только HTTPS `:8333`. Очередь — RabbitMQ с TLS и EXTERNAL; API публикует в exchange `attachment.process`, worker только читает очередь `attachment.process.v1`. + +Образы приложения и worker: non-root uid 10001, `read_only`, `tmpfs /tmp`, `cap_drop: ALL`, `no-new-privileges`. Сторонние образы, которым entrypoint нужен `setuid`/`setgid` (`su-exec`, `setpriv`), запускаются с пустым `cap_drop`: иначе процесс не стартует. Это исключение для postgres, rabbitmq, seaweed, valkey, nginx, OpenBao и telemetry. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдение | Где смотреть | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Контракт не сломан | `python scripts/check_contract.py` | PASS | вывод проверки | +| Публичный smoke по HTTPS | `python tests/smoke.py --base-url https://127.0.0.1:8443 --ca-file secrets/pki/ca.crt` | PASS после рестарта и повторно после ротации leaf, seal и смены S3 secret | `tests/smoke.py` | +| Недоверенный CA | `python tests/security.py` с тем же CA | PASS: пустой trust store отклонён | `tests/security.py` | +| mTLS LB→API, чужой CA, чужая identity, нет обхода :8080 | `docker compose --profile tools run --rm probe` | 29/29 PASS | `app/probes/security.py` | +| OpenBao allow/deny, не root | тот же probe | api читает свой KV и получает 403 на worker и на запись | `infra/openbao/policies/` | +| RabbitMQ mTLS и разные права | тот же probe | API не может `basic_get` (403), worker не может publish (403), чужой CN не логинится | `infra/rabbitmq/` | +| Collector mTLS | тот же probe | API принят, stranger 403, запрос без сертификата 400 | `infra/nginx/otel-edge.conf` | +| Rate limit и размер тела | тот же probe | 429 с `Retry-After`, JSON 70 КБ → 413 | `app/ratelimit.py` | +| Argon2id | измерение в контейнере API | hash 29 мс, verify 25 мс, m=19456, t=2, p=1. Потолок 60 verify/мин ≈ 1.5 с CPU | `evidence/argon2-budget.json` | +| Рестарт без потери данных | `scripts/restart.sh` через Git Bash | 44 с, exit 0. Пользователи/сообщения/ready-вложения те же (9/6/2). OpenBao не перезапускался и остался healthy. Затем smoke снова PASS | `scripts/restart.sh` | +| Отказ одной реплики | `scripts/stop-one.sh` | Остановлен только `api-1`. `/health/ready` отвечал одним оставшимся instance id. Реплика снова стала healthy | `scripts/stop-one.sh` | +| Трек 1, least privilege | `python scripts/track1_run.py` | Узкие права: `password_hash` и canary deny, запись `originals/*` allow. После временного `GRANT SELECT` и Admin в Seaweed те же проверки стали allow. После отзыва снова deny | `evidence/track1-run.json` | +| Отзыв identity и смена S3 secret | тот же сценарий, worker остановлен | Вложение осталось `queued`, в очереди 1 сообщение и после `delete_user`. Старый S3 secret — `SignatureDoesNotMatch`. После восстановления пользователя вложение стало `ready` | `evidence/track1-run.json` | +| Rehash и истечение сессии | `python scripts/verify_local.py` | Хеш `m=8192,t=1,p=1` после входа стал `m=19456,t=2,p=1`. Сессия с `expires_at` в прошлом получила 401 | `evidence/verify-local.json` | +| Ротация leaf LB без пересборки | тот же сценарий | Новый serial: 200 на `api-1:8443`. Старый serial: HTTP 403. После `nginx -s reload` публичный ready 200. Исходный leaf возвращён | `evidence/verify-local.json` | +| Sealed OpenBao | тот же сценарий | До seal неверный пароль даёт 401. Пока vault sealed, login даёт 503. В `printenv` API нет `DATABASE_URL`, ключей S3, `VALKEY_PASSWORD` и `BAO_TOKEN`. После `scripts/unseal.py` ready снова 200 | `evidence/verify-local.json` | +| Секреты не попадают в журналы | подсчёт строк в логах API, worker и Loki за 20 минут | Совпадений `Bearer `, `password=`, `postgresql://`, `aws_secret_access_key` и `BEGIN PRIVATE` нет | `evidence/verify-local.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза трека 1: учётные данные worker достаточны, чтобы обработать вложение, и недостаточны, чтобы читать пароли, писать пользователей, удалять сообщения, читать префикс `canary/` или публиковать в очередь. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Методика: один прогон probe из сети стенда с worker leaf и секретами, которые OpenBao отдаёт этой роли. Значения секретов не печатались, только allow/deny. + +Результат совпал с гипотезой и на узких правах, и после временного расширения. На живом Seaweed worker на время получил `GRANT SELECT ON users` и действия `Admin`/`Read`/`Write`/`List`; `password_hash` и `canary/` стали allow, INSERT пользователя и DELETE сообщений остались `42501`, publish в очередь остался 403. Затем права возвращены, и повторный probe снова совпал с узкой матрицей. Файл `infra/minio/worker-broad.json` описывает ту же идею для MinIO IAM и в рантайм Seaweed не подключается: рабочая политика — `secrets/seaweed/s3.json`. + +Дальше worker был остановлен, вложение осталось `queued`, сообщение лежало в `attachment.process.v1`. Удаление пользователя RabbitMQ сообщение не стёрло. Секрет S3 worker заменён в Seaweed и в OpenBao, контейнер `minio` перезапущен без пересборки образа. Старый секрет получил `SignatureDoesNotMatch`. После возврата пользователя RabbitMQ worker обработал то же вложение до `ready`. + +| Воздействие | Ожидание | Наблюдение | Восстановление | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Рестарт postgres, seaweed, rabbit, valkey, api, worker, nginx | Данные и сессии на томах сохраняются, OpenBao не запечатывается | Счётчики строк не изменились, `/health/ready` 200, smoke PASS | 44 с на сам hook | +| Остановка одной API-реплики | Вторая продолжает отвечать | В ответах остался один instance id | `compose start api-1`, реплика healthy | +| Удаление пользователя RabbitMQ worker при queued-задаче | Сообщение остаётся в очереди, после возврата identity задача доходит до ready | Очередь сохранила 1 сообщение, старый S3 secret отвергнут, вложение стало `ready` | пользователь, права и worker возвращены | +| Временное расширение прав worker | Появляются только те allow, которые выдали | `password_hash` и canary стали allow; INSERT/DELETE и publish остались deny | `REVOKE` и прежний `s3.json`, повторный probe узкий | +| Seal OpenBao | Login перестаёт проверять пароль и не читает секрет из окружения | 401 до seal, 503 пока sealed, секретных имён в окружении нет | `python scripts/unseal.py`, ready 200 | -## Ограничения и следующие шаги +## Ограничения -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Публичный `tests/resilience.py` на Windows не может запустить `.sh` через CreateProcess. Сами hook исполнены Git Bash и отработали. На Linux их нужно вызывать как `ACTION_SCRIPT` для `make test-persistence` и `make test-failover`. + +Ротация leaf и секрета S3 снята отдельными соединениями, без параллельной нагрузки на ingress. Параметры Argon2 оставлены на обязательном минимуме: измеренный verify около 25 мс, а частоту входа режет общий счётчик до вычисления хеша. Разбор зависимостей — в `security/scan-triage.md`. `docker scout` на этой машине требует вход в Docker Hub, поэтому CVE базового слоя образа отдельно не снимались. `pip-audit` по `requirements.txt` выполнен. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..fdeefa7 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger application for lab 6.""" diff --git a/app/asgi.py b/app/asgi.py new file mode 100644 index 0000000..1315c9d --- /dev/null +++ b/app/asgi.py @@ -0,0 +1,124 @@ +import asyncio +import random +import time +import uuid + +from opentelemetry.trace import SpanKind + +from app import faults, http, telemetry +from app.errors import AppError +from app.settings import settings + + +class _TooLarge(Exception): + pass + + +async def _read_body(receive, limit): + chunks = [] + size = 0 + while True: + message = await receive() + if message["type"] == "http.disconnect": + return None + if message["type"] != "http.request": + continue + data = message.get("body", b"") or b"" + size += len(data) + if size > limit: + raise _TooLarge() + chunks.append(data) + if not message.get("more_body"): + return b"".join(chunks) + + +def _too_large(scope, limit): + for name, value in scope.get("headers", []): + if name == b"content-length": + try: + return int(value) > limit + except ValueError: + return True + return False + + +async def app(scope, receive, send): + if scope["type"] == "lifespan": + while True: + message = await receive() + if message["type"] == "lifespan.startup": + telemetry.log_event("service.listening", port=8080) + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await send({"type": "lifespan.shutdown.complete"}) + return + return + if scope["type"] != "http": + return + method = scope.get("method", "GET") + path = scope.get("path", "/") + raw_headers = {} + for name, value in scope.get("headers", []): + raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1") + request_id = http.request_id_from(raw_headers) + template, _name, _groups = http.match(method, path) + limit = http.body_limit(path) + started = time.perf_counter() + status = 500 + response_headers = {} + body = b"" + with telemetry.tracer().start_as_current_span( + f"{method} {template}", + kind=SpanKind.SERVER, + attributes={"http.request.method": method, "http.route": template}, + ) as span: + try: + if _too_large(scope, limit): + raise AppError(413, "payload_too_large", "Слишком большой объект") + try: + payload = await _read_body(receive, limit) + except _TooLarge as exc: + raise AppError(413, "payload_too_large", "Слишком большой объект") from exc + if payload is None: + return + if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload: + raise AppError(413, "payload_too_large", "Слишком большой объект") + query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace")) + if path.startswith("/api/") and random.random() < faults.http_error_ratio(): + raise AppError(500, "internal_error", "Внутренняя ошибка") + + def run(): + return http.handle(method, path, query, raw_headers, payload) + + template, status, response_headers, body = await asyncio.to_thread(run) + span.set_attribute("http.route", template) + except Exception as exc: + status, response_headers, body = http.error_response(exc, request_id) + if not isinstance(exc, AppError): + telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id) + if status >= 500: + telemetry.mark_error(span) + span.set_attribute("error.type", type(exc).__name__) + duration = time.perf_counter() - started + span.set_attribute("http.response.status_code", status) + telemetry.record_http(method, template, status, duration) + telemetry.log_event( + "http.request.completed", + request_id=request_id, + http_route=template, + http_method=method, + status=status, + duration_ms=int(duration * 1000), + ) + header_list = [ + (b"x-request-id", request_id.encode("ascii")), + (b"x-instance-id", settings.instance_id.encode("ascii")), + ] + for key, value in response_headers.items(): + header_list.append((key.lower().encode("latin-1"), value.encode("latin-1"))) + await send({"type": "http.response.start", "status": status, "headers": header_list}) + await send({"type": "http.response.body", "body": body}) + + +def new_request_id(): + return uuid.uuid4().hex diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..3aea6ca --- /dev/null +++ b/app/broker.py @@ -0,0 +1,140 @@ +import json +import ssl +import uuid + +import pika + +from app import telemetry +from app.db import jsonb +from app.settings import settings + +QUEUE = "attachment.process.v1" +EXCHANGE = "attachment.process" + + +def _params(): + ctx = ssl.create_default_context(cafile=settings.tls_ca) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(settings.client_cert, settings.client_key) + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + return pika.ConnectionParameters( + host=settings.broker_host, + port=settings.broker_port, + virtual_host="/", + credentials=pika.credentials.ExternalCredentials(), + ssl_options=pika.SSLOptions(ctx, settings.broker_host), + heartbeat=60, + blocked_connection_timeout=2, + socket_timeout=3, + connection_attempts=1, + retry_delay=0, + ) + + +def init(): + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + finally: + connection.close() + + +def ready(): + try: + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True, passive=True) + finally: + connection.close() + return True + except Exception: + return False + + +def insert_outbox(conn, payload, created_at): + conn.execute( + """ + INSERT INTO outbox (job_id, attachment_id, payload, created_at) + VALUES (%s, %s, %s, %s) + """, + (uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at), + ) + + +def publish_pending(conn, job_id=None): + query = """ + SELECT job_id, payload + FROM outbox + WHERE published_at IS NULL + """ + args = [] + if job_id is not None: + query += " AND job_id = %s" + args.append(job_id) + query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20" + rows = conn.execute(query, args).fetchall() + for row in rows: + payload = row["payload"] + if isinstance(payload, str): + payload = json.loads(payload) + _publish(payload) + conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],)) + return len(rows) + + +def _publish(payload): + carrier = {} + if payload.get("traceparent"): + carrier["traceparent"] = payload["traceparent"] + if payload.get("tracestate"): + carrier["tracestate"] = payload["tracestate"] + parent = telemetry.extract_context(carrier) + with telemetry.tracer().start_as_current_span( + "attachment.process publish", + context=parent, + kind=telemetry.trace.SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation.type": "publish", + }, + ): + outgoing = telemetry.inject_carrier() + body = dict(payload) + if outgoing.get("traceparent"): + body["traceparent"] = outgoing["traceparent"] + if outgoing.get("tracestate"): + body["tracestate"] = outgoing["tracestate"] + _send(body, outgoing) + telemetry.log_event( + "job.published", + job_id=str(payload.get("job_id")), + attempt=int(payload.get("attempt") or 1), + ) + + +def _send(payload, headers): + connection = pika.BlockingConnection(_params()) + try: + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( + exchange=EXCHANGE, + routing_key=QUEUE, + body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers, + ), + mandatory=True, + ) + finally: + connection.close() + + +def consumer_count(): + raise RuntimeError("management api is not used") diff --git a/app/codec.py b/app/codec.py new file mode 100644 index 0000000..22f86b4 --- /dev/null +++ b/app/codec.py @@ -0,0 +1,139 @@ +import base64 +import datetime as dt +import hashlib +import json +import re +import uuid + +from app.errors import AppError, bad_request + +USERNAME = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def utcnow(): + return dt.datetime.now(dt.timezone.utc) + + +def iso(value): + if value.tzinfo is None: + value = value.replace(tzinfo=dt.timezone.utc) + value = value.astimezone(dt.timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z" + + +def parse_iso(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise bad_request() + try: + return dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise bad_request() from exc + + +def parse_uuid(value): + if not isinstance(value, str): + raise bad_request() + try: + return str(uuid.UUID(value)) + except ValueError as exc: + raise bad_request() from exc + + +def require_text(value, minimum, maximum, allow_blank=False): + if not isinstance(value, str): + raise bad_request() + if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value): + raise bad_request() + if not allow_blank and not value.strip(): + raise bad_request() + if not (minimum <= len(value) <= maximum): + raise bad_request() + return value + + +def canonical_message(text, attachment_ids): + body = {"attachment_ids": list(attachment_ids), "text": text} + raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + return hashlib.sha256(raw.encode("utf-8")).digest() + + +def encode_cursor(kind, resource_id, created_at, row_id): + raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(token, kind, resource_id): + if not isinstance(token, str) or not token: + raise bad_request() + padded = token + ("=" * (-len(token) % 4)) + try: + raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3) + created_at = parse_iso(stamp) + row_uuid = str(uuid.UUID(row_id)) + except (ValueError, UnicodeError, AppError) as exc: + raise bad_request() from exc + if cursor_kind != kind or cursor_resource != resource_id: + raise bad_request() + return created_at, row_uuid + + +def fitted_size(width, height, box=256): + if width < 1 or height < 1: + raise ValueError("dimensions") + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, (height * box) // width) + return max(1, (width * box) // height), box + + +def detect_image(data, declared): + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + else: + return None + if declared != detected: + return None + return detected + + +def media_type(header): + if not header or not isinstance(header, str): + return "" + return header.split(";", 1)[0].strip().lower() + + +def loads_object(raw): + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + raise bad_request() from exc + + def pairs(items): + obj = {} + for key, value in items: + if key in obj: + raise ValueError("duplicate") + obj[key] = value + return obj + + try: + value = json.loads(text, object_pairs_hook=pairs) + except (ValueError, UnicodeError) as exc: + raise bad_request() from exc + if not isinstance(value, dict): + raise bad_request() + return value + + +def expect_fields(data, allowed, required): + if set(data) - set(allowed): + raise bad_request() + for key in required: + if key not in data: + raise bad_request() diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..a478aef --- /dev/null +++ b/app/db.py @@ -0,0 +1,98 @@ +import time +from contextlib import contextmanager +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool, PoolTimeout + +from app import telemetry +from app.errors import AppError, unavailable +from app.faults import db_fail, db_slow_seconds +from app.settings import settings + +pool = None + + +def init(): + global pool + pool = ConnectionPool( + settings.database_url, + min_size=1, + max_size=settings.pool_max, + timeout=2, + max_waiting=16, + check=ConnectionPool.check_connection, + kwargs={ + "row_factory": dict_row, + "autocommit": False, + "connect_timeout": 3, + "options": "-c timezone=UTC -c statement_timeout=4000", + }, + open=True, + reconnect_timeout=2, + ) + + +def migrate(): + script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8") + statements = [] + buffer = [] + for line in script.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("--"): + continue + buffer.append(line) + if stripped.endswith(";"): + statements.append("\n".join(buffer)) + buffer = [] + with pool.connection() as conn: + conn.execute("SELECT pg_advisory_lock(861006)") + for statement in statements: + conn.execute(statement) + conn.execute("SELECT pg_advisory_unlock(861006)") + conn.commit() + + +def jsonb(value): + return Jsonb(value) + + +@contextmanager +def connection(): + if db_fail(): + raise unavailable() + delay = db_slow_seconds() + started = time.perf_counter() + span = telemetry.tracer().start_as_current_span( + "db.transaction", + attributes={"db.system": "postgresql"}, + ) + span.__enter__() + try: + if delay: + time.sleep(delay) + with pool.connection() as conn: + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + except AppError: + raise + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc: + raise unavailable() from exc + finally: + span.__exit__(None, None, None) + telemetry.record_db(time.perf_counter() - started) + + +def stats(): + if pool is None: + return 0, 0 + data = pool.get_stats() + used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0)) + waiting = int(data.get("requests_waiting", 0)) + return max(used, 0), waiting diff --git a/app/domain.py b/app/domain.py new file mode 100644 index 0000000..54dcedd --- /dev/null +++ b/app/domain.py @@ -0,0 +1,611 @@ +import datetime as dt +import hashlib +import uuid + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from argon2.low_level import Type +from psycopg.errors import UniqueViolation + +from app import ratelimit + +from app import broker, storage, telemetry +from app.codec import ( + USERNAME, + IDEMPOTENCY, + canonical_message, + decode_cursor, + detect_image, + encode_cursor, + expect_fields, + iso, + parse_uuid, + require_text, + utcnow, +) +from app.db import connection, jsonb +from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unavailable, unsupported +from app.images import validate_image +from app.settings import settings + +BASIC = 'Basic realm="messenger", charset="UTF-8"' +BEARER = 'Bearer realm="messenger"' +_hasher = None +_dummy = None + + +def init_hasher(): + global _hasher, _dummy + _hasher = PasswordHasher( + time_cost=settings.argon2_iterations, + memory_cost=settings.argon2_memory_kib, + parallelism=settings.argon2_parallelism, + hash_len=32, + salt_len=16, + type=Type.ID, + ) + _dummy = _hasher.hash("not-a-real-password") + + +def _user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } + + +def _token_hash(token): + return hashlib.sha256(token.encode("utf-8")).digest() + + +def _password(value): + if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128): + raise bad_request() + return value + + +def as_uuid(value): + if isinstance(value, uuid.UUID): + return value + return uuid.UUID(str(value)) + + +def _username(value): + if not isinstance(value, str) or USERNAME.fullmatch(value) is None: + raise bad_request() + return value + + +def register(data): + expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"]) + username = _username(data["username"]) + display_name = require_text(data["display_name"], 1, 100) + password = _password(data["password"]) + password_hash = _hasher.hash(password) + user_id = uuid.uuid4() + created_at = utcnow() + try: + with connection() as conn: + conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (user_id, username, display_name, password_hash, created_at), + ) + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + except UniqueViolation as exc: + raise conflict("Имя пользователя занято") from exc + return 201, _user_json(row) + + +def login(username, password): + from app.secrets_loader import healthy + + if not healthy(): + raise unavailable() + if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password: + raise unauthorized(BASIC) + if len(password) > 128 or len(username) > 128: + raise unauthorized(BASIC) + ratelimit.enforce_login(username) + with connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s", + (username,), + ).fetchone() + stored = row["password_hash"] if row else _dummy + try: + _hasher.verify(stored, password) + valid = row is not None + except (VerifyMismatchError, VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC) + if _hasher.check_needs_rehash(stored): + with connection() as conn: + conn.execute( + "UPDATE users SET password_hash = %s WHERE id = %s", + (_hasher.hash(password), row["id"]), + ) + token = secrets_token() + expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl) + with connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (_token_hash(token), row["id"], expires_at), + ) + return 201, { + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires_at), + "user": _user_json(row), + } + + +def secrets_token(): + import secrets + + return secrets.token_urlsafe(32) + + +def current_user(token): + if not token or len(token) < 32: + raise unauthorized(BEARER) + with connection() as conn: + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW() + """, + (_token_hash(token),), + ).fetchone() + if not row: + raise unauthorized(BEARER) + return row + + +def ready_check(): + from app.secrets_loader import healthy + + if not healthy(): + raise unavailable() + with connection() as conn: + conn.execute("SELECT 1") + if not storage.ready() or not broker.ready() or not ratelimit.ready(): + raise unavailable() + + +def revoke(token): + user = current_user(token) + with connection() as conn: + conn.execute( + "UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL", + (_token_hash(token),), + ) + return user + + +def create_chat(user, data): + expect_fields(data, ["title", "member_ids"], ["title", "member_ids"]) + title = require_text(data["title"], 1, 100) + raw_members = data["member_ids"] + if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100): + raise bad_request() + members = [parse_uuid(item) for item in raw_members] + if len(set(members)) != len(members): + raise bad_request() + if str(user["id"]) not in members: + raise bad_request() + chat_id = uuid.uuid4() + created_at = utcnow() + with connection() as conn: + found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall() + if len(found) != len(members): + raise bad_request() + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created_at), + ) + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + (chat_id, as_uuid(member_id), position), + ) + return 201, { + "id": str(chat_id), + "title": title, + "member_ids": members, + "created_at": iso(created_at), + } + + +def _chat_json(conn, row): + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position", + (row["id"],), + ).fetchall() + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item["user_id"]) for item in members], + "created_at": iso(row["created_at"]), + } + + +def list_chats(user, limit, cursor): + created_at = None + row_id = None + if cursor is not None: + created_at, row_id = decode_cursor(cursor, "chats", str(user["id"])) + with connection() as conn: + if created_at is None: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at, c.id + LIMIT %s + """, + (user["id"], limit + 1), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (user["id"], created_at, as_uuid(row_id), limit + 1), + ).fetchall() + extra = len(rows) > limit + rows = rows[:limit] + items = [_chat_json(conn, row) for row in rows] + next_cursor = None + if extra and rows: + last = rows[-1] + next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + +def get_chat(user, chat_id): + with connection() as conn: + row = _visible_chat(conn, user["id"], chat_id) + return _chat_json(conn, row) + + +def _visible_chat(conn, user_id, chat_id): + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s AND m.user_id = %s + """, + (as_uuid(chat_id), as_uuid(user_id)), + ).fetchone() + if not row: + raise not_found() + return row + + +def _message_json(row, attachment_ids): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + "attachment_ids": attachment_ids, + } + + +def _attachment_ids(conn, message_id): + rows = conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ).fetchall() + return [str(row["attachment_id"]) for row in rows] + + +def create_message(user, chat_id, key, data, path): + chat_uuid = as_uuid(chat_id) + if IDEMPOTENCY.fullmatch(key or "") is None: + raise bad_request() + expect_fields(data, ["text", "attachment_ids"], ["text"]) + if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000: + raise bad_request() + text = data["text"] + if "attachment_ids" in data: + raw_ids = data["attachment_ids"] + if not isinstance(raw_ids, list) or len(raw_ids) > 10: + raise bad_request() + attachment_ids = [parse_uuid(item) for item in raw_ids] + if len(set(attachment_ids)) != len(attachment_ids): + raise bad_request() + else: + attachment_ids = [] + if not text.strip() and not attachment_ids: + raise bad_request() + body_hash = canonical_message(text, attachment_ids) + lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True) + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,)) + existing = conn.execute( + """ + SELECT body_hash, response_json, created_at + FROM idempotency + WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s + """, + (user["id"], path, key), + ).fetchone() + if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl): + if bytes(existing["body_hash"]) != body_hash: + raise conflict("Ключ идемпотентности уже использован с другим телом") + stored = existing["response_json"] + if isinstance(stored, str): + import json + + stored = json.loads(stored) + return 201, stored + if existing: + conn.execute( + "DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s", + (user["id"], path, key), + ) + if attachment_ids: + rows = conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)", + ([as_uuid(item) for item in attachment_ids],), + ).fetchall() + found = {str(row["id"]): row for row in rows} + if len(found) != len(attachment_ids): + raise not_found() + for attachment_id in attachment_ids: + row = found[attachment_id] + if str(row["chat_id"]) != chat_id: + raise not_found() + if row["status"] == "failed": + raise conflict("Нельзя прикрепить неуспешное вложение") + message_id = uuid.uuid4() + created_at = utcnow() + conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (message_id, chat_uuid, user["id"], text, created_at), + ) + for position, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, as_uuid(attachment_id), position), + ) + stored = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", + (message_id,), + ).fetchone() + payload = _message_json(stored, attachment_ids) + conn.execute( + """ + INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at) + VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s) + """, + (user["id"], path, key, body_hash, jsonb(payload), created_at), + ) + return 201, payload + + +def list_messages(user, chat_id, limit, cursor): + chat_uuid = as_uuid(chat_id) + created_at = None + row_id = None + if cursor is not None: + created_at, row_id = decode_cursor(cursor, "messages", chat_id) + with connection() as conn: + _visible_chat(conn, user["id"], chat_uuid) + if created_at is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at, id + LIMIT %s + """, + (chat_uuid, limit + 1), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT %s + """, + (chat_uuid, created_at, as_uuid(row_id), limit + 1), + ).fetchall() + extra = len(rows) > limit + rows = rows[:limit] + items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows] + next_cursor = None + if extra and rows: + last = rows[-1] + next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + +def _variants(conn, attachment_id): + rows = conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = 1 + ORDER BY name + """, + (attachment_id,), + ).fetchall() + return [ + { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + for row in rows + ] + + +def _attachment_json(row, variants): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": variants, + "error_code": row["error_code"], + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +def upload(user, chat_id, filename, content_type, data): + import hashlib as hl + + if not data: + raise unsupported() + if len(data) > settings.max_upload: + raise too_large() + detected = detect_image(data, content_type) + if not detected: + raise unsupported() + validate_image(data) + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + attachment_id = uuid.uuid4() + created_at = utcnow() + digest = hl.sha256(data).hexdigest() + staging_key = f"staging/{attachment_id}" + storage.put(staging_key, data, detected, "staging") + carrier = telemetry.inject_carrier() + job_id = uuid.uuid4() + payload = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": iso(created_at), + } + if carrier.get("traceparent"): + payload["traceparent"] = carrier["traceparent"] + if carrier.get("tracestate"): + payload["tracestate"] = carrier["tracestate"] + row = { + "id": attachment_id, + "chat_id": chat_id, + "owner_id": user["id"], + "filename": filename, + "content_type": detected, + "size_bytes": len(data), + "sha256": digest, + "status": "queued", + "error_code": None, + "created_at": created_at, + "updated_at": created_at, + } + with connection() as conn: + _visible_chat(conn, user["id"], chat_id) + conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, + error_code, staging_key, pipeline_version, attempt, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s) + """, + ( + attachment_id, + as_uuid(chat_id), + user["id"], + filename, + detected, + len(data), + digest, + staging_key, + created_at, + created_at, + ), + ) + broker.insert_outbox(conn, payload, created_at) + try: + with connection() as conn: + broker.publish_pending(conn, job_id) + except Exception: + telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id)) + telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected) + return 202, _attachment_json(row, []) + + +def get_attachment(user, attachment_id): + attachment_uuid = as_uuid(attachment_id) + with connection() as conn: + row = _visible_attachment(conn, user["id"], attachment_uuid) + return _attachment_json(row, _variants(conn, attachment_uuid)) + + +def _visible_attachment(conn, user_id, attachment_id): + row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone() + if not row: + raise not_found() + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (row["chat_id"], user_id), + ).fetchone() + if not member: + raise not_found() + return row + + +def download(user, attachment_id, variant): + attachment_uuid = as_uuid(attachment_id) + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise bad_request() + with connection() as conn: + row = _visible_attachment(conn, user["id"], attachment_uuid) + if row["status"] != "ready": + raise conflict("Вложение ещё не готово") + if variant == "original": + key = row["original_key"] + content_type = row["content_type"] + else: + found = conn.execute( + """ + SELECT object_key, content_type + FROM attachment_variants + WHERE attachment_id = %s AND name = %s AND pipeline_version = 1 + """, + (attachment_uuid, variant), + ).fetchone() + if not found: + raise not_found() + key = found["object_key"] + content_type = found["content_type"] + return storage.get(key, variant), content_type diff --git a/app/entrypoint.py b/app/entrypoint.py new file mode 100644 index 0000000..d9d688c --- /dev/null +++ b/app/entrypoint.py @@ -0,0 +1,32 @@ +import os +import secrets +from pathlib import Path + + +def _assign_instance(): + alias = secrets.token_hex(8) + target = os.environ.get("INSTANCE_FILE") + if target: + path = Path(target) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(alias, encoding="ascii") + os.environ["INSTANCE_ID"] = alias + + +def main(): + _assign_instance() + from app.secrets_loader import hydrate + + hydrate() + from app.settings import load + + load() + if os.environ.get("APP_ROLE") == "worker": + from app.worker import main as run + else: + from app.main import main as run + run() + + +if __name__ == "__main__": + main() diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 0000000..5e2e187 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,49 @@ +class AppError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message="Неверный запрос"): + return AppError(400, "invalid_request", message) + + +def unauthorized(challenge): + return AppError( + 401, + "unauthorized", + "Требуется аутентификация", + {"WWW-Authenticate": challenge}, + ) + + +def not_found(): + return AppError(404, "not_found", "Объект не найден") + + +def conflict(message="Конфликт"): + return AppError(409, "conflict", message) + + +def too_large(message="Слишком большой объект"): + return AppError(413, "payload_too_large", message) + + +def unsupported(message="Неподдерживаемый тип изображения"): + return AppError(415, "unsupported_media_type", message) + + +def unavailable(message="Сервис временно недоступен"): + return AppError(503, "unavailable", message) + + +def rate_limited(retry_after): + return AppError( + 429, + "rate_limited", + "Слишком много попыток входа", + {"Retry-After": str(int(retry_after))}, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 0000000..5e91539 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,73 @@ +from pathlib import Path + +from app.settings import settings + + +def _text(name): + path = Path(settings.fault_dir) / name + try: + return path.read_text(encoding="utf-8").strip() + except (FileNotFoundError, OSError): + return "" + + +def db_slow_seconds(): + raw = _text("db_slow_ms") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 10000.0) / 1000.0 + + +def db_fail(): + return _text("db_fail") == "1" + + +def http_error_ratio(): + raw = _text("http_error_ratio") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 1.0) + + +def handler_slow_seconds(): + raw = _text("handler_slow_ms") + if not raw: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + if value <= 0: + return 0.0 + return min(value, 10000.0) / 1000.0 + + +def consume_process_failure(): + path = Path(settings.fault_dir) / "process_fail_remaining" + try: + raw = path.read_text(encoding="utf-8").strip() + except (FileNotFoundError, OSError): + return False + try: + remaining = int(raw or "0") + except ValueError: + return False + if remaining <= 0: + return False + try: + path.write_text(str(remaining - 1), encoding="utf-8") + except OSError: + return False + return True diff --git a/app/http.py b/app/http.py new file mode 100644 index 0000000..fd2d564 --- /dev/null +++ b/app/http.py @@ -0,0 +1,221 @@ +import json +import re +from urllib.parse import parse_qs + +from app import domain, faults +from app.codec import REQUEST_ID, loads_object, media_type, require_text +from app.errors import AppError, bad_request, not_found, unauthorized +from app.settings import settings + +_ROUTES = [ + ("GET", re.compile(r"^/health/live$"), "/health/live", "live"), + ("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"), + ("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"), + ("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"), + ("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"), + ("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"), + ("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"), + ("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"), + ("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"), + ("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"), + ("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"), +] + + +def match(method, path): + for route_method, pattern, template, name in _ROUTES: + found = pattern.fullmatch(path) + if found and route_method == method: + return template, name, found.groups() + return "unmatched", "unmatched", () + + +def _single(query, name): + values = query.get(name) + if not values: + return None + if len(values) != 1 or values[0] == "": + raise bad_request() + return values[0] + + +def _limit(query): + raw = _single(query, "limit") + if raw is None: + return 50 + if re.fullmatch(r"[1-9][0-9]*", raw) is None: + raise bad_request() + value = int(raw) + if value > 100: + raise bad_request() + return value + + +def _json_body(headers, body): + if media_type(headers.get("content-type", "")) != "application/json": + raise bad_request() + if not body: + raise bad_request() + return loads_object(body) + + +def _bearer(headers): + value = headers.get("authorization", "") + if not value.lower().startswith("bearer "): + raise unauthorized(domain.BEARER) + token = value.split(" ", 1)[1].strip() + return token + + +def _basic(headers): + import base64 + + value = headers.get("authorization", "") + if not value.lower().startswith("basic "): + raise unauthorized(domain.BASIC) + try: + decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise unauthorized(domain.BASIC) from exc + if ":" not in decoded: + raise unauthorized(domain.BASIC) + return decoded.split(":", 1) + + +def _filename(query): + raw = _single(query, "filename") + if raw is None: + raise bad_request() + if len(raw) < 1 or len(raw) > 255 or not raw.strip(): + raise bad_request() + if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw): + raise bad_request() + return raw + + +def handle(method, path, query, headers, body): + template, name, groups = match(method, path) + if name == "unmatched": + raise not_found() + if name not in {"live", "ready"}: + delay = faults.handler_slow_seconds() + if delay: + import time + + time.sleep(delay) + if name == "live": + return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"}) + if name == "ready": + domain.ready_check() + return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"}) + if name == "register": + status, payload = domain.register(_json_body(headers, body)) + return template, status, _json_headers(), _dump(payload) + if name == "login": + if body: + raise bad_request() + username, password = _basic(headers) + status, payload = domain.login(username, password) + return template, status, _json_headers(), _dump(payload) + if name == "me": + user = domain.current_user(_bearer(headers)) + return template, 200, _json_headers(), _dump(domain._user_json(user)) + if name == "logout": + domain.revoke(_bearer(headers)) + return template, 204, {}, b"" + if name == "create_chat": + user = domain.current_user(_bearer(headers)) + status, payload = domain.create_chat(user, _json_body(headers, body)) + return template, status, _json_headers(), _dump(payload) + if name == "list_chats": + user = domain.current_user(_bearer(headers)) + payload = domain.list_chats(user, _limit(query), _single(query, "cursor")) + return template, 200, _json_headers(), _dump(payload) + if name == "get_chat": + user = domain.current_user(_bearer(headers)) + payload = domain.get_chat(user, _uuid_segment(groups[0])) + return template, 200, _json_headers(), _dump(payload) + if name == "create_message": + user = domain.current_user(_bearer(headers)) + chat_id = _uuid_segment(groups[0]) + status, payload = domain.create_message( + user, + chat_id, + headers.get("idempotency-key"), + _json_body(headers, body), + path, + ) + return template, status, _json_headers(), _dump(payload) + if name == "list_messages": + user = domain.current_user(_bearer(headers)) + payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor")) + return template, 200, _json_headers(), _dump(payload) + if name == "upload": + user = domain.current_user(_bearer(headers)) + content_type = media_type(headers.get("content-type", "")) + status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body) + return template, status, _json_headers(), _dump(payload) + if name == "get_attachment": + user = domain.current_user(_bearer(headers)) + payload = domain.get_attachment(user, _uuid_segment(groups[0])) + return template, 200, _json_headers(), _dump(payload) + if name == "download": + user = domain.current_user(_bearer(headers)) + variant = _single(query, "variant") or "original" + data, content_type = domain.download(user, _uuid_segment(groups[0]), variant) + return template, 200, {"Content-Type": content_type}, data + raise not_found() + + +def _uuid_segment(value): + from app.codec import parse_uuid + + return parse_uuid(value) + + +def _json_headers(): + return {"Content-Type": "application/json; charset=utf-8"} + + +def _dump(payload): + return json.dumps(payload, ensure_ascii=False).encode("utf-8") + + +def error_response(exc, request_id): + if isinstance(exc, AppError): + status = exc.status + code = exc.code + message = exc.message + headers = dict(exc.headers) + else: + status = 500 + code = "internal_error" + message = "Внутренняя ошибка" + headers = {} + headers["Content-Type"] = "application/json; charset=utf-8" + body = _dump({"error": {"code": code, "message": message, "request_id": request_id}}) + return status, headers, body + + +def request_id_from(headers): + incoming = headers.get("x-request-id", "") + if REQUEST_ID.fullmatch(incoming): + return incoming + import uuid + + return uuid.uuid4().hex + + +def query_from(raw): + if not raw: + return {} + return parse_qs(raw, keep_blank_values=True, strict_parsing=False) + + +def body_limit(path): + if "/attachments" in path and path.rstrip("/").endswith("attachments"): + return settings.max_upload + return settings.json_max_bytes diff --git a/app/images.py b/app/images.py new file mode 100644 index 0000000..b4e940a --- /dev/null +++ b/app/images.py @@ -0,0 +1,62 @@ +import io + +from PIL import Image, ImageOps, UnidentifiedImageError + +from app.codec import fitted_size +from app.errors import AppError, too_large, unsupported +from app.settings import settings + +Image.MAX_IMAGE_PIXELS = settings.max_pixels + + +def _open(data): + try: + with Image.open(io.BytesIO(data)) as image: + image.verify() + with Image.open(io.BytesIO(data)) as image: + width, height = image.size + if width < 1 or height < 1 or width * height > settings.max_pixels: + raise too_large() + image.load() + return image.copy() + except AppError: + raise + except Image.DecompressionBombError as exc: + raise too_large() from exc + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc: + raise unsupported() from exc + + +def validate_image(data): + _open(data).close() + + +def _rgb(image): + if image.mode == "RGB": + return image + if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + + +def thumbnail_jpeg(data): + image = _open(data) + try: + image = ImageOps.exif_transpose(image) + image = _rgb(image) + target = fitted_size(image.width, image.height, 256) + if (image.width, image.height) != target: + image = image.resize(target, Image.Resampling.LANCZOS) + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=85, optimize=True) + blob = buffer.getvalue() + return blob, image.width, image.height + except AppError: + raise + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise unsupported() from exc + finally: + image.close() diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..ef32529 --- /dev/null +++ b/app/main.py @@ -0,0 +1,53 @@ +import threading +import time + +import uvicorn + +from app import broker, db, domain, ratelimit, storage, telemetry +from app.asgi import app +from app.mtls_proxy import serve_forever +from app.poller import start +from app.settings import settings + + +def _retry(label, action): + last = None + for _ in range(40): + try: + action() + return + except Exception as exc: + last = exc + telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__) + time.sleep(1) + raise last + + +def main(): + telemetry.init() + ratelimit.init() + _retry("postgres", db.init) + _retry("s3", storage.init) + _retry("broker", broker.init) + domain.init_hasher() + start(publish=True) + threading.Thread(target=serve_forever, name="mtls", daemon=True).start() + uvicorn.run( + app, + host="127.0.0.1", + port=settings.local_port, + access_log=False, + log_config={ + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "uvicorn": {"level": "WARNING"}, + "uvicorn.error": {"level": "WARNING"}, + "uvicorn.access": {"level": "WARNING"}, + }, + }, + ) + + +if __name__ == "__main__": + main() diff --git a/app/mtls_proxy.py b/app/mtls_proxy.py new file mode 100644 index 0000000..16e9f05 --- /dev/null +++ b/app/mtls_proxy.py @@ -0,0 +1,109 @@ +"""Terminate TLS in the API process and allow only the LB leaf serials.""" +import asyncio +import ssl + +from app.settings import require + + +def _serials(): + settings = require() + try: + raw = open(settings.allowed_serials_file, "r", encoding="utf-8").read() + except OSError: + return set() + found = set() + for line in raw.splitlines(): + item = line.strip().replace(":", "") + if not item or item.startswith("#"): + continue + try: + found.add(format(int(item, 16), "X")) + except ValueError: + continue + return found + + +def _allowed(sslobj): + settings = require() + cert = sslobj.getpeercert() + if not cert: + return False + names = [value for kind, value in cert.get("subjectAltName", ()) if kind == "DNS"] + if settings.allowed_client_san not in names: + return False + serial = str(cert.get("serialNumber") or "") + try: + normalized = format(int(serial.replace(":", ""), 16), "X") + except ValueError: + return False + return normalized in _serials() + + +async def _pipe(reader, writer): + try: + while True: + data = await reader.read(65536) + if not data: + break + writer.write(data) + await writer.drain() + except Exception: + pass + finally: + try: + writer.close() + except Exception: + pass + + +async def _reject(writer): + body = b'{"error":"client_identity_rejected"}' + head = ( + b"HTTP/1.1 403 Forbidden\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(body)).encode("ascii") + b"\r\n" + b"Connection: close\r\n\r\n" + ) + writer.write(head + body) + try: + await writer.drain() + except Exception: + pass + try: + writer.close() + except Exception: + pass + + +async def _handle(reader, writer): + sslobj = writer.get_extra_info("ssl_object") + if sslobj is None or not _allowed(sslobj): + await _reject(writer) + return + try: + local_reader, local_writer = await asyncio.open_connection("127.0.0.1", require().local_port) + except Exception: + writer.close() + return + await asyncio.gather(_pipe(reader, local_writer), _pipe(local_reader, writer)) + + +def _context(): + settings = require() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(settings.tls_cert, settings.tls_key) + ctx.load_verify_locations(settings.tls_ca) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = False + return ctx + + +async def _serve(): + server = await asyncio.start_server(_handle, "0.0.0.0", require().mtls_port, ssl=_context()) + async with server: + await server.serve_forever() + + +def serve_forever(): + asyncio.run(_serve()) diff --git a/app/poller.py b/app/poller.py new file mode 100644 index 0000000..d83e27f --- /dev/null +++ b/app/poller.py @@ -0,0 +1,42 @@ +import threading + +from app import broker, db, telemetry + + +def _once(publish=True): + try: + with db.connection() as conn: + row = conn.execute( + """ + SELECT COUNT(*) AS depth, + EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age + FROM attachments + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + depth = int(row["depth"] or 0) + age = float(row["age"] or 0) if depth else 0.0 + used, waiting = db.stats() + telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting) + except Exception: + telemetry.update_runtime(db_up=0) + telemetry.sample_process() + if not publish: + return + try: + with db.connection() as conn: + broker.publish_pending(conn) + except Exception: + telemetry.log_event("job.publish_deferred", severity="WARNING") + + +def start(publish=True): + stop = threading.Event() + + def loop(): + while not stop.wait(2): + _once(publish) + + _once(publish) + threading.Thread(target=loop, name="poller", daemon=True).start() + return stop diff --git a/app/probes/__init__.py b/app/probes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/probes/security.py b/app/probes/security.py new file mode 100644 index 0000000..b85dc33 --- /dev/null +++ b/app/probes/security.py @@ -0,0 +1,331 @@ +"""Negative checks for the trust matrix. Prints statuses only, never secret values.""" +import json +import socket +import ssl +import urllib.error +import urllib.request +import uuid + +import pika + + +CA = "/probe/ca.crt" +RESULTS = [] + + +def record(name, ok, detail): + RESULTS.append({"name": name, "ok": bool(ok), "detail": detail}) + print(("PASS " if ok else "FAIL ") + name + " " + detail) + + +def tls_context(cert=None, key=None, ca=CA): + ctx = ssl.create_default_context(cafile=ca) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + if cert and key: + ctx.load_cert_chain(cert, key) + return ctx + + +def http(url, cert=None, key=None, ca=CA, data=None, method="GET", headers=None, hostname=None): + ctx = tls_context(cert, key, ca) + request = urllib.request.Request(url, data=data, headers=headers or {}, method=method) + return urllib.request.urlopen(request, context=ctx, timeout=8) + + +def _handshake_rejected(exc): + if isinstance(exc, urllib.error.URLError) and exc.reason is not None: + exc = exc.reason + if isinstance(exc, (ssl.SSLError, ConnectionResetError, ConnectionAbortedError, BrokenPipeError)): + return True + text = str(exc).lower() + return "certificate" in text or "ssl" in text or "tls" in text + + +def expect_ssl(name, func): + try: + func() + except urllib.error.HTTPError as exc: + record(name, exc.code == 400, "http " + str(exc.code)) + return + except Exception as exc: + record(name, _handshake_rejected(exc), type(exc).__name__) + return + record(name, False, "connection succeeded") + + +def api_identity(): + try: + with http("https://api-1:8443/health/ready", "/probe/lb.crt", "/probe/lb.key") as response: + record("lb-identity-allowed", response.status == 200, "http " + str(response.status)) + except Exception as exc: + record("lb-identity-allowed", False, type(exc).__name__) + try: + with http("https://api-1:8443/health/ready", "/probe/worker.crt", "/probe/worker.key") as response: + record("worker-cert-rejected-by-api", response.status == 403, "http " + str(response.status)) + except urllib.error.HTTPError as exc: + record("worker-cert-rejected-by-api", exc.code == 403, "http " + str(exc.code)) + except Exception as exc: + record("worker-cert-rejected-by-api", False, type(exc).__name__) + try: + with http("https://api-1:8443/health/ready", "/probe/stranger.crt", "/probe/stranger.key") as response: + record("stranger-san-rejected", response.status == 403, "http " + str(response.status)) + except urllib.error.HTTPError as exc: + record("stranger-san-rejected", exc.code == 403, "http " + str(exc.code)) + except Exception as exc: + record("stranger-san-rejected", False, type(exc).__name__) + expect_ssl("api-without-client-cert", lambda: http("https://api-1:8443/health/ready")) + expect_ssl( + "api-foreign-ca", + lambda: http("https://api-1:8443/health/ready", "/probe/other.crt", "/probe/other.key", ca="/probe/other-ca.crt"), + ) + expect_ssl("api-expired-leaf", lambda: http("https://api-1:8443/health/ready", "/probe/expired.crt", "/probe/expired.key")) + + +def bypass(): + sock = socket.socket() + sock.settimeout(3) + try: + sock.connect(("api-1", 8080)) + record("api-local-port-not-routable", False, "connected") + except OSError as exc: + record("api-local-port-not-routable", True, type(exc).__name__) + finally: + sock.close() + + +def bao(cert, key): + ctx = tls_context(cert, key) + request = urllib.request.Request( + "https://openbao:8200/v1/auth/cert/login", + data=b"{}", + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, context=ctx, timeout=8) as response: + payload = json.loads(response.read().decode("utf-8")) + token = (payload.get("auth") or {}).get("client_token") + policies = (payload.get("auth") or {}).get("policies") or [] + if not token or "root" in policies: + raise RuntimeError("unexpected token shape") + return token + + +def bao_status(token, path, method="GET", body=None): + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request( + "https://openbao:8200/v1/" + path, + data=data, + headers={"X-Vault-Token": token, "Content-Type": "application/json"}, + method=method, + ) + try: + with urllib.request.urlopen(request, context=tls_context("/probe/api.crt", "/probe/api.key"), timeout=8) as response: + raw = response.read() + return response.status, json.loads(raw.decode("utf-8")) if raw else {} + except urllib.error.HTTPError as exc: + exc.read(256) + return exc.code, {} + + +def openbao_checks(): + try: + token = bao("/probe/api.crt", "/probe/api.key") + status, payload = bao_status(token, "secret/data/api/db") + username = (((payload.get("data") or {}).get("data") or {}).get("username")) + record("api-reads-own-db-secret", status == 200 and username == "api_login", "http " + str(status)) + denied, _ = bao_status(token, "secret/data/worker/db") + record("api-cannot-read-worker-secret", denied == 403, "http " + str(denied)) + written, _ = bao_status(token, "secret/data/api/db", "POST", {"data": {"username": "nope"}}) + record("api-cannot-write-secret", written in {403, 405}, "http " + str(written)) + except Exception as exc: + record("api-openbao-login", False, type(exc).__name__) + try: + token = bao("/probe/worker.crt", "/probe/worker.key") + status, payload = bao_status(token, "secret/data/worker/db") + # bao_status uses the api client cert for TLS, while the token is the worker token. + username = (((payload.get("data") or {}).get("data") or {}).get("username")) + record("worker-reads-own-db-secret", status == 200 and username == "worker_login", "http " + str(status)) + denied, _ = bao_status(token, "secret/data/api/db") + record("worker-cannot-read-api-secret", denied == 403, "http " + str(denied)) + except Exception as exc: + record("worker-openbao-login", False, type(exc).__name__) + try: + bao("/probe/stranger.crt", "/probe/stranger.key") + record("stranger-openbao-login-denied", False, "login succeeded") + except urllib.error.HTTPError as exc: + record("stranger-openbao-login-denied", exc.code in {400, 403}, "http " + str(exc.code)) + except Exception as exc: + record("stranger-openbao-login-denied", False, type(exc).__name__) + expect_ssl("openbao-without-client-cert", lambda: bao(None, None)) + + +def rabbit_params(cert, key): + ctx = tls_context(cert, key) + return pika.ConnectionParameters( + host="rabbitmq", + port=5671, + virtual_host="/", + credentials=pika.credentials.ExternalCredentials(), + ssl_options=pika.SSLOptions(ctx, "rabbitmq"), + socket_timeout=5, + blocked_connection_timeout=5, + connection_attempts=1, + ) + + +def rabbit_checks(): + try: + connection = pika.BlockingConnection(rabbit_params("/probe/api.crt", "/probe/api.key")) + channel = connection.channel() + channel.queue_declare(queue="attachment.process.v1", passive=True) + record("api-broker-mtls", True, "passive declare") + try: + channel.basic_get("attachment.process.v1", auto_ack=False) + record("api-cannot-consume", False, "basic_get succeeded") + except pika.exceptions.ChannelClosedByBroker as exc: + record("api-cannot-consume", exc.reply_code == 403, "reply " + str(exc.reply_code)) + connection.close() + except Exception as exc: + record("api-broker-mtls", False, type(exc).__name__) + try: + connection = pika.BlockingConnection(rabbit_params("/probe/worker.crt", "/probe/worker.key")) + channel = connection.channel() + channel.confirm_delivery() + try: + channel.basic_publish(exchange="", routing_key="attachment.process.v1", body=b"{}") + record("worker-cannot-publish", False, "publish succeeded") + except pika.exceptions.ChannelClosedByBroker as exc: + record("worker-cannot-publish", exc.reply_code == 403, "reply " + str(exc.reply_code)) + except pika.exceptions.AMQPError as exc: + record("worker-cannot-publish", "403" in str(exc) or "access_refused" in str(exc).lower(), type(exc).__name__) + connection.close() + except Exception as exc: + record("worker-cannot-publish", False, type(exc).__name__) + try: + pika.BlockingConnection(rabbit_params("/probe/stranger.crt", "/probe/stranger.key")) + record("stranger-broker-rejected", False, "connected") + except pika.exceptions.AMQPConnectionError: + record("stranger-broker-rejected", True, "amqp auth failed") + except ssl.SSLError as exc: + record("stranger-broker-rejected", False, type(exc).__name__) + except Exception as exc: + record("stranger-broker-rejected", True, type(exc).__name__) + expect_ssl("broker-without-client-cert", lambda: pika.BlockingConnection(rabbit_params(None, None))) + expect_ssl( + "broker-foreign-ca", + lambda: pika.BlockingConnection( + pika.ConnectionParameters( + host="rabbitmq", + port=5671, + credentials=pika.credentials.ExternalCredentials(), + ssl_options=pika.SSLOptions(tls_context("/probe/other.crt", "/probe/other.key", "/probe/other-ca.crt"), "rabbitmq"), + socket_timeout=5, + connection_attempts=1, + ) + ), + ) + + +def otel_checks(): + body = b"{}" + try: + with http("https://otel-edge:4318/v1/traces", "/probe/api.crt", "/probe/api.key", data=body, method="POST", headers={"Content-Type": "application/json"}) as response: + record("api-collector-mtls", response.status < 500, "http " + str(response.status)) + except urllib.error.HTTPError as exc: + record("api-collector-mtls", exc.code not in {401, 403}, "http " + str(exc.code)) + except Exception as exc: + record("api-collector-mtls", False, type(exc).__name__) + try: + with http("https://otel-edge:4318/v1/traces", "/probe/stranger.crt", "/probe/stranger.key", data=body, method="POST", headers={"Content-Type": "application/json"}) as response: + record("stranger-collector-rejected", response.status == 403, "http " + str(response.status)) + except urllib.error.HTTPError as exc: + record("stranger-collector-rejected", exc.code == 403, "http " + str(exc.code)) + except Exception as exc: + record("stranger-collector-rejected", False, type(exc).__name__) + expect_ssl("collector-without-client-cert", lambda: http("https://otel-edge:4318/v1/traces", data=body, method="POST")) + + +def plain_backend(): + sock = socket.socket() + sock.settimeout(3) + try: + sock.connect(("loki", 3100)) + sock.sendall(b"GET /ready HTTP/1.0\r\nHost: loki\r\n\r\n") + data = sock.recv(32) + record("loki-plaintext-rejected", not data.startswith(b"HTTP/1.1 200") and not data.startswith(b"HTTP/1.0 200"), "bytes " + str(len(data))) + except OSError as exc: + record("loki-plaintext-rejected", False, type(exc).__name__) + finally: + sock.close() + try: + with http("https://loki:3100/ready") as response: + record("loki-tls-server-verified", response.status in {200, 503}, "http " + str(response.status)) + except urllib.error.HTTPError as exc: + record("loki-tls-server-verified", exc.code in {200, 401, 403, 503}, "http " + str(exc.code)) + except Exception as exc: + record("loki-tls-server-verified", False, type(exc).__name__) + + +def public_limits(): + try: + with http("https://nginx:8443/health/ready") as response: + record("ingress-trusted-https", response.status == 200, "http " + str(response.status)) + except Exception as exc: + record("ingress-trusted-https", False, type(exc).__name__) + name = "lim" + uuid.uuid4().hex[:8] + password = "correct-horse-1" + body = json.dumps({"username": name, "display_name": "Limit", "password": password}).encode("utf-8") + try: + with http("https://nginx:8443/api/v1/users", data=body, method="POST", headers={"Content-Type": "application/json"}) as response: + record("register-via-ingress", response.status == 201, "http " + str(response.status)) + except Exception as exc: + record("register-via-ingress", False, type(exc).__name__) + return + import base64 + + auth = base64.b64encode((name + ":" + password).encode("utf-8")).decode("ascii") + saw_429 = False + retry = "" + for _ in range(12): + request = urllib.request.Request( + "https://nginx:8443/api/v1/auth/sessions", + headers={"Authorization": "Basic " + auth}, + method="POST", + ) + try: + with urllib.request.urlopen(request, context=tls_context(), timeout=15) as response: + response.read() + except urllib.error.HTTPError as exc: + if exc.code == 429: + saw_429 = True + retry = exc.headers.get("Retry-After") or "" + break + record("login-rate-limit", saw_429 and retry.isdigit(), "retry " + ("set" if retry else "missing")) + huge = json.dumps({"username": "abc", "display_name": "n", "password": "p" * 70000}).encode("utf-8") + try: + http("https://nginx:8443/api/v1/users", data=huge, method="POST", headers={"Content-Type": "application/json"}) + record("json-body-limit", False, "accepted") + except urllib.error.HTTPError as exc: + record("json-body-limit", exc.code == 413, "http " + str(exc.code)) + except Exception as exc: + record("json-body-limit", False, type(exc).__name__) + + +def main(): + api_identity() + bypass() + openbao_checks() + rabbit_checks() + otel_checks() + plain_backend() + public_limits() + failed = [item for item in RESULTS if not item["ok"]] + print("SUMMARY passed=%d failed=%d" % (len(RESULTS) - len(failed), len(failed))) + if failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/app/probes/track1.py b/app/probes/track1.py new file mode 100644 index 0000000..4cb9cd8 --- /dev/null +++ b/app/probes/track1.py @@ -0,0 +1,125 @@ +"""Least-privilege probe for the worker identity. Output contains statuses, not secret values.""" +import json +import ssl +import urllib.request + +import boto3 +import psycopg +from botocore.config import Config +from botocore.exceptions import ClientError +import pika + + +def bao_login(): + ctx = ssl.create_default_context(cafile="/probe/ca.crt") + ctx.load_cert_chain("/probe/worker.crt", "/probe/worker.key") + request = urllib.request.Request( + "https://openbao:8200/v1/auth/cert/login", + data=b"{}", + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, context=ctx, timeout=8) as response: + payload = json.loads(response.read().decode("utf-8")) + return payload["auth"]["client_token"], ctx + + +def read_secret(token, ctx, path): + request = urllib.request.Request( + "https://openbao:8200/v1/secret/data/" + path, + headers={"X-Vault-Token": token}, + ) + with urllib.request.urlopen(request, context=ctx, timeout=8) as response: + payload = json.loads(response.read().decode("utf-8")) + return payload["data"]["data"] + + +def sql(dsn, statement): + try: + with psycopg.connect(dsn, connect_timeout=5, autocommit=True) as conn: + conn.execute(statement) + return "allow" + except Exception as exc: + code = getattr(getattr(exc, "diag", None), "sqlstate", None) or type(exc).__name__ + return "deny:" + str(code) + + +def s3_put(client, key): + try: + client.put_object(Bucket="messenger-attachments", Key=key, Body=b"probe") + return "allow" + except ClientError as exc: + return "deny:" + str(exc.response.get("Error", {}).get("Code", "error")) + + +def s3_get(client, key): + try: + client.get_object(Bucket="messenger-attachments", Key=key)["Body"].read(32) + return "allow" + except ClientError as exc: + return "deny:" + str(exc.response.get("Error", {}).get("Code", "error")) + + +def publish(): + ctx = ssl.create_default_context(cafile="/probe/ca.crt") + ctx.load_cert_chain("/probe/worker.crt", "/probe/worker.key") + params = pika.ConnectionParameters( + host="rabbitmq", + port=5671, + credentials=pika.credentials.ExternalCredentials(), + ssl_options=pika.SSLOptions(ctx, "rabbitmq"), + socket_timeout=5, + connection_attempts=1, + ) + try: + connection = pika.BlockingConnection(params) + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish(exchange="", routing_key="attachment.process.v1", body=b"{}") + connection.close() + return "allow" + except pika.exceptions.ChannelClosedByBroker as exc: + return "deny:" + str(exc.reply_code) + except Exception as exc: + return "deny:" + type(exc).__name__ + + +def main(): + token, ctx = bao_login() + db = read_secret(token, ctx, "worker/db") + s3 = read_secret(token, ctx, "worker/s3") + dsn = "postgresql://{username}:{password}@{host}:5432/{dbname}?sslmode=verify-full&sslrootcert=/probe/ca.crt".format(**db) + client = boto3.client( + "s3", + endpoint_url="https://minio:8333", + aws_access_key_id=s3["access_key"], + aws_secret_access_key=s3["secret_key"], + region_name="us-east-1", + verify="/probe/ca.crt", + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=8), + ) + report = { + "db_select_attachment_metadata": sql(dsn, "SELECT id FROM attachments LIMIT 1"), + "db_select_password_hash": sql(dsn, "SELECT password_hash FROM users LIMIT 1"), + "db_insert_user": sql( + dsn, + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES (gen_random_uuid(), 'evil', 'e', 'x', NOW())", + ), + "db_delete_message": sql(dsn, "DELETE FROM messages WHERE false"), + "s3_put_original": s3_put(client, "originals/privilege-probe"), + "s3_get_canary": s3_get(client, "canary/marker"), + "s3_put_canary": s3_put(client, "canary/owned-by-worker"), + "queue_publish": publish(), + } + print(json.dumps(report, sort_keys=True)) + denied = ["db_select_password_hash", "db_insert_user", "db_delete_message", "s3_get_canary", "s3_put_canary", "queue_publish"] + allowed = ["db_select_attachment_metadata", "s3_put_original"] + bad = [name for name in denied if not str(report[name]).startswith("deny")] + bad += [name for name in allowed if report[name] != "allow"] + if bad: + raise SystemExit("unexpected privilege result") + + +if __name__ == "__main__": + main() diff --git a/app/ratelimit.py b/app/ratelimit.py new file mode 100644 index 0000000..6d64de4 --- /dev/null +++ b/app/ratelimit.py @@ -0,0 +1,61 @@ +import hashlib + +import redis +from redis.exceptions import RedisError + +from app.errors import rate_limited, unavailable +from app.settings import require + +_client = None + + +def init(): + global _client + settings = require() + if settings.role != "api": + return + _client = redis.Redis( + host=settings.valkey_host, + port=settings.valkey_port, + username=settings.valkey_username, + password=settings.valkey_password, + ssl=True, + ssl_ca_certs=settings.tls_ca, + ssl_certfile=settings.client_cert, + ssl_keyfile=settings.client_key, + socket_timeout=1.5, + socket_connect_timeout=1.5, + decode_responses=True, + ) + + +def ready(): + settings = require() + if settings.role != "api": + return True + try: + return bool(_client.ping()) + except RedisError: + return False + + +def _hit(key, limit, window): + count = _client.incr(key) + if count == 1: + _client.expire(key, window) + if int(count) > limit: + retry = _client.ttl(key) + if retry is None or int(retry) < 1: + retry = window + raise rate_limited(int(retry)) + + +def enforce_login(username): + """Shared counter for every API replica. Runs before Argon2id.""" + settings = require() + digest = hashlib.sha256(username.encode("utf-8", "replace")).hexdigest()[:32] + try: + _hit("login:global", settings.login_global_limit, settings.login_window) + _hit("login:" + digest, settings.login_limit, settings.login_window) + except RedisError as exc: + raise unavailable() from exc diff --git a/app/schema.sql b/app/schema.sql new file mode 100644 index 0000000..6ef564f --- /dev/null +++ b/app/schema.sql @@ -0,0 +1,108 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); + +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + sha256 text NOT NULL, + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + staging_key text NOT NULL, + original_key text, + pipeline_version integer NOT NULL DEFAULT 1, + attempt integer NOT NULL DEFAULT 0, + lease_until timestamptz, + worker_id text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + ready_at timestamptz +); + +CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at) + WHERE status IN ('queued', 'processing'); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + name text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + pipeline_version integer NOT NULL, + PRIMARY KEY (attachment_id, name, pipeline_version) +); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); + +CREATE TABLE IF NOT EXISTS idempotency ( + user_id uuid NOT NULL, + method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + body_hash bytea NOT NULL, + status_code integer NOT NULL, + response_json jsonb NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE TABLE IF NOT EXISTS outbox ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz +); + +CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at) + WHERE published_at IS NULL; diff --git a/app/secrets_loader.py b/app/secrets_loader.py new file mode 100644 index 0000000..3530090 --- /dev/null +++ b/app/secrets_loader.py @@ -0,0 +1,137 @@ +"""OpenBao cert-auth login and scoped KV reads. No root token and no token in the environment.""" +import json +import os +import ssl +import threading +import time +import urllib.error +import urllib.request + +_lock = threading.Lock() +_token = None +_lease = 0 +_healthy = False +_reason = "not-loaded" + + +def healthy(): + return _healthy + + +def reason(): + return _reason + + +def _context(): + ctx = ssl.create_default_context(cafile=os.environ["TLS_CA_FILE"]) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(os.environ["CLIENT_CERT_FILE"], os.environ["CLIENT_KEY_FILE"]) + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + return ctx + + +def _request(method, path, body=None, token=None): + data = None if body is None else json.dumps(body).encode("utf-8") + headers = {"Content-Type": "application/json"} + if token: + headers["X-Vault-Token"] = token + request = urllib.request.Request(os.environ["BAO_ADDR"].rstrip("/") + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, context=_context(), timeout=5) as response: + raw = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read(512) + raise RuntimeError("openbao http " + str(exc.code) + " " + path) from exc + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + +def _login(): + global _token, _lease, _healthy, _reason + payload = _request("POST", "/v1/auth/cert/login", {}) + auth = payload.get("auth") or {} + token = auth.get("client_token") + policies = auth.get("policies") or auth.get("token_policies") or [] + if isinstance(policies, str): + policies = [policies] + if not token or not policies: + raise RuntimeError("openbao cert login returned no scoped token") + if "root" in policies: + raise RuntimeError("refusing a root token") + _token = token + _lease = int(auth.get("lease_duration") or 0) + _healthy = True + _reason = "ok" + + +def _read(path): + payload = _request("GET", "/v1/secret/data/" + path, token=_token) + data = ((payload.get("data") or {}).get("data") or {}) + if not data: + raise RuntimeError("openbao empty secret " + path) + return data + + +def _apply(role): + db = _read(role + "/db") + s3 = _read(role + "/s3") + ca = os.environ["TLS_CA_FILE"] + os.environ["DATABASE_URL"] = ( + "postgresql://{username}:{password}@{host}:5432/{dbname}?sslmode=verify-full&sslrootcert={ca}".format( + username=db["username"], + password=db["password"], + host=db["host"], + dbname=db["dbname"], + ca=ca, + ) + ) + os.environ["S3_ACCESS_KEY_ID"] = s3["access_key"] + os.environ["S3_SECRET_ACCESS_KEY"] = s3["secret_key"] + if role == "api": + cache = _read("api/valkey") + os.environ["VALKEY_PASSWORD"] = cache["password"] + os.environ["VALKEY_USERNAME"] = cache.get("username") or "api" + os.environ["VALKEY_HOST"] = cache.get("host") or "valkey" + else: + os.environ["VALKEY_PASSWORD"] = "" + + +def _renew_loop(): + global _healthy, _reason, _lease + while True: + time.sleep(15) + try: + with _lock: + payload = _request("POST", "/v1/auth/token/renew-self", {}, token=_token) + auth = payload.get("auth") or {} + _lease = int(auth.get("lease_duration") or _lease or 0) + _healthy = True + _reason = "ok" + except Exception: + try: + with _lock: + _login() + except Exception as exc: + _healthy = False + _reason = type(exc).__name__ + + +def hydrate(): + """Authenticate with the service leaf and pull only that role's KV paths.""" + role = os.environ.get("APP_ROLE", "api") + if role not in {"api", "worker"}: + raise RuntimeError("unsupported role") + last = None + for _ in range(40): + try: + with _lock: + _login() + _apply(role) + threading.Thread(target=_renew_loop, name="bao-renew", daemon=True).start() + return + except Exception as exc: + last = exc + time.sleep(2) + raise last diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 0000000..4c2ace3 --- /dev/null +++ b/app/settings.py @@ -0,0 +1,88 @@ +import os +from pathlib import Path + + +def _env(name, default=None): + value = os.environ.get(name) + if value is not None and value.strip() != "": + return value.strip() + if default is not None: + return default + raise RuntimeError("missing configuration " + name) + + +class Settings: + def __init__(self): + self.service_name = _env("OTEL_SERVICE_NAME", "messenger-api") + self.service_version = _env("SERVICE_VERSION", "1.7.0") + self.instance_id = _env("INSTANCE_ID", "unset") + self.environment = "lab" + self.role = _env("APP_ROLE", "api") + self.database_url = _env("DATABASE_URL") + self.session_ttl = int(_env("SESSION_TTL_SECONDS", "3600")) + self.idempotency_ttl = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.s3_endpoint = _env("S3_ENDPOINT", "https://minio:8333") + self.s3_region = _env("S3_REGION", "us-east-1") + self.s3_bucket = _env("S3_BUCKET", "messenger-attachments") + self.s3_access_key = _env("S3_ACCESS_KEY_ID") + self.s3_secret_key = _env("S3_SECRET_ACCESS_KEY") + self.broker_host = _env("BROKER_HOST", "rabbitmq") + self.broker_port = int(_env("BROKER_PORT", "5671")) + self.worker_concurrency = int(_env("WORKER_CONCURRENCY", "2")) + self.job_max_attempts = int(_env("JOB_MAX_ATTEMPTS", "5")) + self.job_timeout = int(_env("JOB_TIMEOUT_SECONDS", "60")) + self.staging_retention = int(_env("STAGING_RETENTION_SECONDS", "86400")) + self.max_upload = int(_env("MAX_UPLOAD_BYTES", "10485760")) + self.max_pixels = int(_env("MAX_IMAGE_PIXELS", "40000000")) + self.json_max_bytes = int(_env("JSON_MAX_BYTES", "65536")) + self.pool_max = int(_env("DB_POOL_MAX", "4")) + self.otlp_endpoint = _env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel-edge:4318").rstrip("/") + self.fault_dir = _env("FAULT_DIR", "/faults") + self.tls_ca = _env("TLS_CA_FILE", "/certs/ca.crt") + self.tls_cert = _env("TLS_CERT_FILE", "/certs/server.crt") + self.tls_key = _env("TLS_KEY_FILE", "/certs/server.key") + self.client_cert = _env("CLIENT_CERT_FILE", "/certs/client.crt") + self.client_key = _env("CLIENT_KEY_FILE", "/certs/client.key") + self.allowed_client_san = _env("ALLOWED_CLIENT_SAN", "lb.messenger.internal") + self.allowed_serials_file = _env("ALLOWED_SERIALS_FILE", "/certs/lb-allowed-serials.txt") + self.mtls_port = int(_env("MTLS_PORT", "8443")) + self.local_port = int(_env("APP_PORT", "8080")) + self.argon2_memory_kib = max(int(_env("ARGON2_MEMORY_KIB", "19456")), 19456) + self.argon2_iterations = max(int(_env("ARGON2_ITERATIONS", "2")), 2) + self.argon2_parallelism = max(int(_env("ARGON2_PARALLELISM", "1")), 1) + self.login_limit = int(_env("LOGIN_LIMIT", "8")) + self.login_window = int(_env("LOGIN_WINDOW_SECONDS", "60")) + self.login_global_limit = int(_env("LOGIN_GLOBAL_LIMIT", "30")) + self.valkey_host = _env("VALKEY_HOST", "valkey") + self.valkey_port = int(_env("VALKEY_PORT", "6379")) + self.valkey_username = _env("VALKEY_USERNAME", "api") + self.valkey_password = _env("VALKEY_PASSWORD", "") + self.bao_addr = _env("BAO_ADDR", "https://openbao:8200") + for key in ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "VALKEY_PASSWORD"): + os.environ.pop(key, None) + + +_state = {"current": None} + + +class _Proxy: + def __getattr__(self, name): + current = _state["current"] + if current is None: + raise RuntimeError("settings are not loaded") + return getattr(current, name) + + +def load(): + _state["current"] = Settings() + return _state["current"] + + +settings = _Proxy() + + +def require(): + current = _state["current"] + if current is None: + raise RuntimeError("settings are not loaded") + return current diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..c14e118 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,70 @@ +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from app import telemetry +from app.settings import settings + +_client = None +_lock = threading.Lock() + + +def _client_for(): + return boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + aws_access_key_id=settings.s3_access_key, + aws_secret_access_key=settings.s3_secret_key, + region_name=settings.s3_region, + verify=settings.tls_ca, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + ), + ) + + +def init(): + global _client + _client = _client_for() + prefix = "staging/" if settings.role == "api" else "originals/" + _client.list_objects_v2(Bucket=settings.s3_bucket, Prefix=prefix, MaxKeys=1) + + +def ready(): + try: + prefix = "staging/" if settings.role == "api" else "originals/" + with _lock: + _client.list_objects_v2(Bucket=settings.s3_bucket, Prefix=prefix, MaxKeys=1) + return True + except (BotoCoreError, ClientError, OSError): + return False + + +def put(key, data, content_type, variant): + with telemetry.tracer().start_as_current_span( + "s3.put", + attributes={"s3.operation": "put", "variant": variant}, + ): + with _lock: + _client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type) + + +def get(key, variant): + with telemetry.tracer().start_as_current_span( + "s3.get", + attributes={"s3.operation": "get", "variant": variant}, + ): + with _lock: + response = _client.get_object(Bucket=settings.s3_bucket, Key=key) + return response["Body"].read() + + +def delete(key): + with _lock: + _client.delete_object(Bucket=settings.s3_bucket, Key=key) diff --git a/app/telemetry.py b/app/telemetry.py new file mode 100644 index 0000000..631819b --- /dev/null +++ b/app/telemetry.py @@ -0,0 +1,389 @@ +import json +import logging +import threading +import time + +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.propagate import extract, inject +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk._logs.export import LogExportResult +from opentelemetry.sdk.metrics.export import MetricExportResult +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult +from opentelemetry.trace import Status, StatusCode + +from app.settings import settings + +BANNED_FIELDS = { + "password", + "token", + "authorization", + "secret", + "filename", + "text", + "body", + "database_url", + "broker_url", + "image", +} + +_dropped = None +_http_requests = None +_http_duration = None +_jobs = None +_job_duration = None +_ready_duration = None +_db_duration = None +_state = { + "queue_depth": 0, + "queue_age": 0.0, + "db_up": 0, + "pool_used": 0, + "pool_waiting": 0, + "worker_busy": 0, + "worker_capacity": 0, + "worker_consumers": 0, + "cpu": 0.0, + "memory": 0, +} +_cpu_last = None +_lock = threading.Lock() + + +class _GuardedExporter: + def __init__(self, inner, signal): + self.inner = inner + self.signal = signal + + def __getattr__(self, name): + return getattr(self.inner, name) + + def export(self, items, **kwargs): + try: + result = self.inner.export(items) + except Exception: + _count_drop(self.signal, len(items)) + return self._failure() + if getattr(result, "name", None) != "SUCCESS": + _count_drop(self.signal, len(items)) + return result + + def _failure(self): + if self.signal == "metrics": + return MetricExportResult.FAILURE + if self.signal == "logs": + return LogExportResult.FAILURE + return SpanExportResult.FAILURE + + def shutdown(self, **kwargs): + try: + return self.inner.shutdown(**kwargs) + except Exception: + return None + + def force_flush(self, timeout_millis=3000): + try: + return self.inner.force_flush(timeout_millis) + except Exception: + return False + + +class _RawFormatter(logging.Formatter): + def format(self, record): + return record.getMessage() + + +def _count_drop(signal, count): + if _dropped is not None and count: + _dropped.add(count, {"signal": signal}) + + +def _resource(): + return Resource.create( + { + "service.name": settings.service_name, + "service.version": settings.service_version, + "service.instance.id": settings.instance_id, + "deployment.environment.name": settings.environment, + } + ) + + +def _histogram(name, buckets): + return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets)) + + +def _otlp(endpoint): + return dict( + endpoint=endpoint, + timeout=5, + certificate_file=settings.tls_ca, + client_certificate_file=settings.client_cert, + client_key_file=settings.client_key, + ) + + +def init(): + global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration + resource = _resource() + endpoint = settings.otlp_endpoint + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor( + _GuardedExporter(OTLPSpanExporter(**_otlp(endpoint + "/v1/traces")), "spans"), + max_queue_size=256, + schedule_delay_millis=2000, + max_export_batch_size=64, + export_timeout_millis=5000, + ) + ) + trace.set_tracer_provider(tracer_provider) + + reader = PeriodicExportingMetricReader( + _GuardedExporter(OTLPMetricExporter(**_otlp(endpoint + "/v1/metrics")), "metrics"), + export_interval_millis=5000, + export_timeout_millis=4000, + ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[reader], + views=[ + _histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]), + _histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]), + _histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]), + _histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]), + ], + ) + from opentelemetry import metrics + + metrics.set_meter_provider(meter_provider) + meter = metrics.get_meter("messenger", settings.service_version) + _dropped = meter.create_counter("messenger.telemetry.dropped", unit="1") + _http_requests = meter.create_counter("http.server.requests", unit="1") + _http_duration = meter.create_histogram("http.server.duration", unit="s") + _jobs = meter.create_counter("messenger.jobs", unit="1") + _job_duration = meter.create_histogram("messenger.job.duration", unit="s") + _ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s") + _db_duration = meter.create_histogram("messenger.db.duration", unit="s") + meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth]) + meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s") + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting]) + meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy]) + meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity]) + meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers]) + meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up]) + meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu]) + meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By") + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + _GuardedExporter(OTLPLogExporter(**_otlp(endpoint + "/v1/logs")), "logs"), + max_queue_size=256, + schedule_delay_millis=2000, + max_export_batch_size=64, + export_timeout_millis=5000, + ) + ) + root = logging.getLogger() + root.handlers.clear() + root.setLevel(logging.INFO) + stream = logging.StreamHandler() + stream.setFormatter(_RawFormatter()) + root.addHandler(stream) + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root.addHandler(otel_handler) + for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access", "redis"): + logging.getLogger(name).setLevel(logging.WARNING) + quiet = logging.getLogger("opentelemetry") + quiet.handlers.clear() + quiet.propagate = False + quiet.setLevel(logging.CRITICAL) + log_event("service.started", role=settings.role) + + +def tracer(): + return trace.get_tracer("messenger", settings.service_version) + + +def current_ids(): + span = trace.get_current_span() + ctx = span.get_span_context() + if not ctx or not ctx.is_valid: + return None, None + return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def log_event(event, severity="INFO", **fields): + trace_id, span_id = current_ids() + payload = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z", + "severity": severity, + "service.name": settings.service_name, + "event": event, + "trace_id": trace_id, + "span_id": span_id, + } + for key, value in fields.items(): + if key in BANNED_FIELDS or value is None: + continue + if isinstance(value, (str, int, float, bool)): + payload[key] = value + level = getattr(logging, severity, logging.INFO) + logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False)) + + +def inject_carrier(carrier=None): + target = {} if carrier is None else carrier + inject(target) + return target + + +def extract_context(carrier): + clean = {} + for key, value in (carrier or {}).items(): + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + if isinstance(value, str): + clean[str(key).lower()] = value + return extract(clean) + + +def record_http(method, route, status, duration): + if _http_requests is None: + return + klass = f"{int(status) // 100}xx" + attributes = { + "http.request.method": method, + "http.route": route, + "http.response.status_class": klass, + } + _http_requests.add(1, attributes) + _http_duration.record(duration, attributes) + + +def record_job(outcome, duration): + attributes = {"job_type": "attachment.process.v1", "outcome": outcome} + if _jobs is not None: + _jobs.add(1, attributes) + _job_duration.record(max(duration, 0.0), attributes) + + +def record_ready(duration): + if _ready_duration is not None: + _ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"}) + + +def record_db(duration): + if _db_duration is not None: + _db_duration.record(max(duration, 0.0), {"db.system": "postgresql"}) + + +def update_runtime(**values): + with _lock: + _state.update(values) + + +def note_worker(busy, capacity): + update_runtime(worker_busy=busy, worker_capacity=capacity) + + +def sample_process(): + global _cpu_last + now_cpu = time.process_time() + now_wall = time.monotonic() + ratio = 0.0 + if _cpu_last is not None: + dc = now_cpu - _cpu_last[0] + dw = now_wall - _cpu_last[1] + if dw > 0: + ratio = dc / dw + _cpu_last = (now_cpu, now_wall) + memory = 0 + try: + with open("/proc/self/statm", encoding="ascii") as handle: + resident = int(handle.read().split()[1]) + memory = resident * os_page() + except (OSError, IndexError, ValueError): + memory = 0 + update_runtime(cpu=ratio, memory=memory) + + +_page = None + + +def os_page(): + global _page + if _page is None: + import os + + _page = os.sysconf("SC_PAGE_SIZE") + return _page + + +def _obs(value, attributes=None): + from opentelemetry.metrics import Observation + + return [Observation(value, attributes or {})] + + +def _queue_depth(_options): + return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"}) + + +def _queue_age(_options): + return _obs(_state["queue_age"], {"queue": "attachment.process.v1"}) + + +def _pool_used(_options): + return _obs(_state["pool_used"], {"pool": "postgres"}) + + +def _pool_waiting(_options): + return _obs(_state["pool_waiting"], {"pool": "postgres"}) + + +def _worker_busy(_options): + if settings.role != "worker": + return [] + return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"}) + + +def _worker_capacity(_options): + if settings.role != "worker": + return [] + return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"}) + + +def _worker_consumers(_options): + return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"}) + + +def _db_up(_options): + return _obs(_state["db_up"]) + + +def _cpu(_options): + return _obs(_state["cpu"]) + + +def _memory(_options): + return _obs(_state["memory"]) + + +def mark_error(span): + span.set_status(Status(StatusCode.ERROR)) + + +def attach(ctx): + return otel_context.attach(ctx) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..9f35539 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,348 @@ +import datetime as dt +import hashlib +import json +import signal +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pika +from botocore.exceptions import BotoCoreError, ClientError +from opentelemetry.trace import SpanKind + +from app import broker, db, storage, telemetry +from app.broker import QUEUE, _params +from app.codec import utcnow +from app.errors import AppError +from app.faults import consume_process_failure +from app.images import thumbnail_jpeg +from app.poller import start +from app.settings import settings + +_inflight = 0 +_inflight_lock = threading.Lock() +_stop = threading.Event() + + +class Transient(Exception): + pass + + +class Permanent(Exception): + pass + + +@contextmanager +def _busy(): + global _inflight + with _inflight_lock: + _inflight += 1 + current = _inflight + telemetry.note_worker(current, settings.worker_concurrency) + try: + yield + finally: + with _inflight_lock: + _inflight -= 1 + current = _inflight + telemetry.note_worker(current, settings.worker_concurrency) + + +def _mark_failed(attachment_id, attempt, code): + with db.connection() as conn: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (code, utcnow(), attachment_id, attempt), + ) + telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code) + + +def _release(attachment_id, attempt): + with db.connection() as conn: + conn.execute( + """ + UPDATE attachments + SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (utcnow(), attachment_id, attempt), + ) + + +def process_delivery(body, headers): + started = time.perf_counter() + try: + payload = json.loads(body.decode("utf-8")) + attachment_id = uuid.UUID(str(payload["attachment_id"])) + pipeline = int(payload["pipeline_version"]) + except (ValueError, UnicodeError, KeyError, TypeError): + telemetry.log_event("job.poison", severity="ERROR") + return "ack" + carrier = {} + for key, value in (headers or {}).items(): + if isinstance(value, bytes): + value = value.decode("utf-8", "replace") + if isinstance(value, str): + carrier[str(key).lower()] = value + if payload.get("traceparent"): + carrier.setdefault("traceparent", payload["traceparent"]) + if payload.get("tracestate"): + carrier.setdefault("tracestate", payload["tracestate"]) + parent = telemetry.extract_context(carrier) + outcome = None + with telemetry.tracer().start_as_current_span( + "attachment.process consume", + context=parent, + kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": QUEUE, + "messaging.operation.type": "consume", + }, + ) as span: + try: + action, outcome, attempt = _consume(payload, attachment_id, pipeline, span) + except Exception: + telemetry.mark_error(span) + telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id)) + action, outcome = "requeue", "retry" + attempt = None + if outcome: + telemetry.record_job(outcome, time.perf_counter() - started) + if attempt is not None: + span.set_attribute("attempt", attempt) + if outcome in {"failed", "retry"}: + telemetry.mark_error(span) + return action + + +def _consume(payload, attachment_id, pipeline, span): + now = utcnow() + with db.connection() as conn: + row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() + if not row or int(row["pipeline_version"]) != pipeline: + return "ack", None, None + if row["status"] == "ready": + return "ack", None, int(row["attempt"]) + if row["status"] == "failed": + return "ack", None, int(row["attempt"]) + if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now: + held_attempt = int(row["attempt"]) + staging_key = None + else: + held_attempt = None + attempt = int(row["attempt"]) + 1 + if attempt > settings.job_max_attempts: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s + WHERE id = %s + """, + (now, attachment_id), + ) + return "ack", "failed", attempt + lease = now + dt.timedelta(seconds=settings.job_timeout) + conn.execute( + """ + UPDATE attachments + SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s + WHERE id = %s + """, + (attempt, lease, settings.instance_id, now, attachment_id), + ) + staging_key = row["staging_key"] + content_type = row["content_type"] + expected_sha = row["sha256"] + created_at = row["created_at"] + if staging_key is None: + span.add_event("lease_held") + time.sleep(1) + return "requeue", None, held_attempt + span.set_attribute("attempt", attempt) + telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) + try: + if consume_process_failure(): + span.add_event("retry", {"reason": "injected"}) + raise Transient() + data = storage.get(staging_key, "staging") + if hashlib.sha256(data).hexdigest() != expected_sha: + raise Permanent("invalid_image") + thumb, width, height = thumbnail_jpeg(data) + original_key = f"originals/{attachment_id}" + thumb_key = f"variants/{attachment_id}/thumbnail" + storage.put(original_key, data, content_type, "original") + storage.put(thumb_key, thumb, "image/jpeg", "thumbnail") + except Permanent as exc: + _mark_failed(attachment_id, attempt, str(exc) or "processing_failed") + return "ack", "failed", attempt + except AppError as exc: + code = "invalid_image" if exc.status in {413, 415} else "processing_failed" + _mark_failed(attachment_id, attempt, code) + return "ack", "failed", attempt + except Transient: + _release(attachment_id, attempt) + telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient") + time.sleep(0.5) + return "requeue", "retry", attempt + except (BotoCoreError, ClientError, OSError, TimeoutError): + _release(attachment_id, attempt) + telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage") + time.sleep(min(2 ** max(attempt - 1, 0), 8)) + return "requeue", "retry", attempt + thumb_sha = hashlib.sha256(thumb).hexdigest() + ready_at = utcnow() + with db.connection() as conn: + current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() + if not current or current["status"] != "processing" or int(current["attempt"]) != attempt: + return "ack", None, attempt + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version + ) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE + SET content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + """, + (attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s, + lease_until = NULL, updated_at = %s + WHERE id = %s AND status = 'processing' AND attempt = %s + """, + (original_key, ready_at, ready_at, attachment_id, attempt), + ) + try: + storage.delete(staging_key) + except (BotoCoreError, ClientError, OSError): + telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id)) + telemetry.record_ready((ready_at - created_at).total_seconds()) + telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id"))) + return "ack", "success", attempt + + +def _serve(connection): + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.basic_qos(prefetch_count=settings.worker_concurrency) + executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job") + + def on_message(ch, method, properties, body): + def work(): + with _busy(): + try: + action = process_delivery(body, properties.headers if properties else None) + except Exception: + action = "requeue" + + def finish(): + try: + if action == "ack": + ch.basic_ack(method.delivery_tag) + else: + ch.basic_nack(method.delivery_tag, requeue=True) + except Exception: + telemetry.log_event("job.ack_failed", severity="ERROR") + + connection.add_callback_threadsafe(finish) + + executor.submit(work) + + channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False) + + def watch(): + _stop.wait() + connection.add_callback_threadsafe(channel.stop_consuming) + + threading.Thread(target=watch, name="stop-watch", daemon=True).start() + channel.start_consuming() + executor.shutdown(wait=True, cancel_futures=False) + + +class _Health(BaseHTTPRequestHandler): + def do_GET(self): + if self.path.startswith("/health/live"): + self._send(200) + return + if self.path.startswith("/health/ready"): + try: + from app.secrets_loader import healthy + + with db.connection() as conn: + conn.execute("SELECT 1") + ok = healthy() and storage.ready() and broker.ready() + except Exception: + ok = False + self._send(200 if ok else 503) + return + self._send(404) + + def _send(self, status): + body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}' + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + return + + +def _health_server(): + server = ThreadingHTTPServer(("127.0.0.1", 8081), _Health) + server.serve_forever() + + +def _request_stop(signum, _frame): + _stop.set() + + +def main(): + signal.signal(signal.SIGTERM, _request_stop) + signal.signal(signal.SIGINT, _request_stop) + telemetry.init() + from app.main import _retry + + _retry("postgres", db.init) + _retry("s3", storage.init) + _retry("broker", broker.init) + telemetry.note_worker(0, settings.worker_concurrency) + start(publish=False) + threading.Thread(target=_health_server, name="health", daemon=True).start() + while not _stop.is_set(): + connection = None + try: + connection = pika.BlockingConnection(_params()) + telemetry.update_runtime(worker_consumers=settings.worker_concurrency) + _serve(connection) + except Exception: + if _stop.is_set(): + break + telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection") + time.sleep(1) + finally: + if connection is not None and connection.is_open: + try: + connection.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2c86892 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,533 @@ +name: mtusi-lab7 + +x-hardening: &hardening + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + +x-app-env: &app-env + SERVICE_VERSION: "1.7.0" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: https://minio:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + BROKER_HOST: rabbitmq + BROKER_PORT: "5671" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + JSON_MAX_BYTES: "65536" + DB_POOL_MAX: "4" + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-edge:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + TLS_CA_FILE: /certs/ca.crt + ARGON2_MEMORY_KIB: "19456" + ARGON2_ITERATIONS: "2" + ARGON2_PARALLELISM: "1" + LOGIN_LIMIT: "10" + LOGIN_WINDOW_SECONDS: "60" + LOGIN_GLOBAL_LIMIT: "60" + BAO_ADDR: https://openbao:8200 + VALKEY_HOST: valkey + VALKEY_PORT: "6379" + FAULT_DIR: /faults + AWS_EC2_METADATA_DISABLED: "true" + +services: + postgres: + <<: *hardening + # Upstream entrypoint uses gosu and therefore needs SETUID/SETGID. + cap_drop: [] + build: ./infra/postgres + image: mtusi-lab7-postgres:16.6 + env_file: + - secrets/postgres.env + volumes: + - pg-data:/var/lib/postgresql/data + - ./secrets/db/01-roles.sql:/docker-entrypoint-initdb.d/01-roles.sql:ro + - ./app/schema.sql:/docker-entrypoint-initdb.d/02-schema.sql:ro + - ./infra/postgres/03-grants.sql:/docker-entrypoint-initdb.d/03-grants.sql:ro + - ./infra/postgres/pg_hba.conf:/etc/postgres/pg_hba.conf:ro + - ./secrets/pki/ca.crt:/certs-src/ca.crt:ro + - ./secrets/pki/postgres.crt:/certs-src/server.crt:ro + - ./secrets/pki/postgres.key:/certs-src/server.key:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 15s + mem_limit: 384m + cpus: "0.75" + pids_limit: 128 + networks: [app] + restart: unless-stopped + + minio: + <<: *hardening + # Image entrypoint drops root with su-exec and needs SETUID/SETGID. + cap_drop: [] + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -filer=true + - -s3 + - -s3.port=0 + - -s3.port.https=8333 + - -s3.port.grpc=0 + - -s3.iam=false + - -s3.ip.bind=0.0.0.0 + - -s3.cert.file=/certs/public.crt + - -s3.key.file=/certs/private.key + - -s3.config=/etc/seaweed/s3.json + - -s3.port.iceberg=0 + - -s3.port.lance=0 + volumes: + - s3-data:/data + - ./secrets/pki/minio:/certs:ro + - ./secrets/seaweed/s3.json:/etc/seaweed/s3.json:ro + healthcheck: + test: ["CMD", "weed", "version"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 8s + mem_limit: 384m + cpus: "0.50" + pids_limit: 128 + networks: [app] + restart: unless-stopped + + rabbitmq: + <<: *hardening + # Upstream entrypoint uses su-exec and therefore needs SETUID/SETGID. + cap_drop: [] + build: ./infra/rabbitmq + image: mtusi-lab7-rabbitmq:3.13.7 + hostname: rabbitmq + volumes: + - rabbit-data:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + - ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro + - ./secrets/pki/ca.crt:/certs-src/ca.crt:ro + - ./secrets/pki/rabbitmq.crt:/certs-src/server.crt:ro + - ./secrets/pki/rabbitmq.key:/certs-src/server.key:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + mem_limit: 384m + cpus: "0.50" + pids_limit: 256 + networks: [app] + restart: unless-stopped + + valkey: + <<: *hardening + # Image entrypoint uses setpriv to drop root and needs SETUID/SETGID. + cap_drop: [] + image: valkey/valkey:8.0.2 + command: ["valkey-server", "/etc/valkey/valkey.conf"] + volumes: + - valkey-data:/data + - ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro + - ./secrets/valkey/users.acl:/etc/valkey/users.acl:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/valkey.crt:/certs/server.crt:ro + - ./secrets/pki/valkey.key:/certs/server.key:ro + healthcheck: + test: ["CMD", "valkey-cli", "--tls", "--cacert", "/certs/ca.crt", "-h", "valkey", "--user", "health", "ping"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 5s + mem_limit: 128m + cpus: "0.25" + pids_limit: 64 + networks: [app] + restart: unless-stopped + + openbao-init: + image: openbao/openbao:2.2.0 + user: "0:0" + entrypoint: ["sh", "-c", "chown -R 100:1000 /openbao/data && chmod 750 /openbao/data"] + volumes: + - bao-data:/openbao/data + security_opt: + - no-new-privileges:true + networks: [app] + restart: "no" + + openbao: + <<: *hardening + image: openbao/openbao:2.2.0 + user: "100:1000" + entrypoint: ["bao"] + command: ["server", "-config=/openbao/config.hcl"] + depends_on: + openbao-init: + condition: service_completed_successfully + volumes: + - bao-data:/openbao/data + - ./infra/openbao/config.hcl:/openbao/config.hcl:ro + - ./infra/openbao/policies:/openbao/policies:ro + - ./secrets/pki/ca.crt:/openbao/tls/ca.crt:ro + - ./secrets/pki/openbao.crt:/openbao/tls/server.crt:ro + - ./secrets/pki/openbao.key:/openbao/tls/server.key:ro + - ./secrets/pki/health.crt:/openbao/tls/health.crt:ro + - ./secrets/pki/health.key:/openbao/tls/health.key:ro + - ./secrets/pki/operator.crt:/openbao/tls/operator.crt:ro + - ./secrets/pki/operator.key:/openbao/tls/operator.key:ro + healthcheck: + test: + [ + "CMD", + "bao", + "status", + "-address=https://127.0.0.1:8200", + "-ca-cert=/openbao/tls/ca.crt", + "-client-cert=/openbao/tls/health.crt", + "-client-key=/openbao/tls/health.key", + ] + interval: 5s + timeout: 5s + retries: 40 + start_period: 5s + mem_limit: 256m + cpus: "0.30" + pids_limit: 64 + networks: [app] + restart: unless-stopped + + api-1: &api + <<: *hardening + build: . + image: mtusi-lab7-messenger:1.7.0 + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-api + APP_ROLE: api + INSTANCE_FILE: /instance/id + TLS_CERT_FILE: /certs/server.crt + TLS_KEY_FILE: /certs/server.key + CLIENT_CERT_FILE: /certs/client.crt + CLIENT_KEY_FILE: /certs/client.key + ALLOWED_CLIENT_SAN: lb.messenger.internal + ALLOWED_SERIALS_FILE: /certs/lb-allowed-serials.txt + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.7.0 + volumes: + - ./run/instances/api-1:/instance + - ./faults:/faults:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/api-server.crt:/certs/server.crt:ro + - ./secrets/pki/api-server.key:/certs/server.key:ro + - ./secrets/pki/api-client.crt:/certs/client.crt:ro + - ./secrets/pki/api-client.key:/certs/client.key:ro + - ./secrets/pki/lb-allowed-serials.txt:/certs/lb-allowed-serials.txt:ro + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + rabbitmq: + condition: service_healthy + valkey: + condition: service_healthy + openbao: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 40 + start_period: 25s + mem_limit: 384m + cpus: "0.75" + pids_limit: 128 + networks: [app] + restart: unless-stopped + + api-2: + <<: *api + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-api + APP_ROLE: api + INSTANCE_FILE: /instance/id + TLS_CERT_FILE: /certs/server.crt + TLS_KEY_FILE: /certs/server.key + CLIENT_CERT_FILE: /certs/client.crt + CLIENT_KEY_FILE: /certs/client.key + ALLOWED_CLIENT_SAN: lb.messenger.internal + ALLOWED_SERIALS_FILE: /certs/lb-allowed-serials.txt + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.7.0 + volumes: + - ./run/instances/api-2:/instance + - ./faults:/faults:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/api-server.crt:/certs/server.crt:ro + - ./secrets/pki/api-server.key:/certs/server.key:ro + - ./secrets/pki/api-client.crt:/certs/client.crt:ro + - ./secrets/pki/api-client.key:/certs/client.key:ro + - ./secrets/pki/lb-allowed-serials.txt:/certs/lb-allowed-serials.txt:ro + + worker: + <<: *hardening + build: . + image: mtusi-lab7-messenger:1.7.0 + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + command: ["python", "-m", "app.entrypoint"] + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-worker + APP_ROLE: worker + INSTANCE_FILE: /instance/id + TLS_CERT_FILE: /certs/client.crt + TLS_KEY_FILE: /certs/client.key + CLIENT_CERT_FILE: /certs/client.crt + CLIENT_KEY_FILE: /certs/client.key + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.7.0 + volumes: + - ./run/instances/worker:/instance + - ./faults:/faults:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/worker-client.crt:/certs/client.crt:ro + - ./secrets/pki/worker-client.key:/certs/client.key:ro + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + rabbitmq: + condition: service_healthy + openbao: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=3)"] + interval: 5s + timeout: 4s + retries: 40 + start_period: 20s + mem_limit: 320m + cpus: "0.50" + pids_limit: 64 + networks: [app] + restart: unless-stopped + + nginx: + <<: *hardening + cap_drop: [] + image: nginx:1.27.4-alpine + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + ports: + - "127.0.0.1:8443:8443" + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx/ingress.conf:/etc/nginx/conf.d/ingress.conf:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/ingress.crt:/certs/ingress.crt:ro + - ./secrets/pki/ingress.key:/certs/ingress.key:ro + - ./secrets/pki/lb.crt:/certs/lb.crt:ro + - ./secrets/pki/lb.key:/certs/lb.key:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + mem_limit: 64m + cpus: "0.25" + pids_limit: 32 + networks: [app] + restart: unless-stopped + + otel-edge: + <<: *hardening + cap_drop: [] + image: nginx:1.27.4-alpine + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx/otel-edge.conf:/etc/nginx/conf.d/otel.conf:ro + - ./secrets/nginx/serials.conf:/etc/nginx/serials.conf:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/otel-edge.crt:/certs/server.crt:ro + - ./secrets/pki/otel-edge.key:/certs/server.key:ro + mem_limit: 64m + cpus: "0.20" + pids_limit: 32 + networks: [app, telemetry] + restart: unless-stopped + + otel-collector: + <<: *hardening + cap_drop: [] + image: otel/opentelemetry-collector-contrib:0.120.0 + command: ["--config=/etc/otelcol/collector.yaml"] + read_only: true + tmpfs: + - /tmp + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + - ./secrets/pki/ca.crt:/certs/ca.crt:ro + - ./secrets/pki/collector.crt:/certs/collector.crt:ro + - ./secrets/pki/collector.key:/certs/collector.key:ro + depends_on: + - tempo + - loki + mem_limit: 256m + cpus: "0.40" + pids_limit: 64 + networks: [telemetry] + restart: unless-stopped + + prometheus: + <<: *hardening + cap_drop: [] + image: prom/prometheus:v2.55.1 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=2d + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro + - ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro + - ./secrets/pki/ca.crt:/etc/prometheus/ca.crt:ro + - ./secrets/pki/prometheus.crt:/certs/prometheus.crt:ro + - ./secrets/pki/prometheus.key:/certs/prometheus.key:ro + - prometheus-data:/prometheus + mem_limit: 256m + cpus: "0.30" + pids_limit: 64 + networks: [telemetry] + restart: unless-stopped + + loki: + <<: *hardening + cap_drop: [] + image: grafana/loki:3.3.2 + command: ["-config.file=/etc/loki/loki.yml"] + volumes: + - ./infra/loki/loki.yml:/etc/loki/loki.yml:ro + - ./secrets/pki/loki.crt:/certs/loki.crt:ro + - ./secrets/pki/loki.key:/certs/loki.key:ro + - loki-data:/loki + mem_limit: 256m + cpus: "0.30" + pids_limit: 64 + networks: [telemetry] + restart: unless-stopped + + tempo: + <<: *hardening + cap_drop: [] + image: grafana/tempo:2.7.2 + command: ["-config.file=/etc/tempo/tempo.yml"] + volumes: + - ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro + - ./secrets/pki/tempo.crt:/certs/tempo.crt:ro + - ./secrets/pki/tempo.key:/certs/tempo.key:ro + - tempo-data:/var/tempo + mem_limit: 256m + cpus: "0.30" + pids_limit: 64 + networks: [telemetry] + restart: unless-stopped + + grafana: + <<: *hardening + cap_drop: [] + image: grafana/grafana:11.4.0 + environment: + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /etc/grafana/certs/grafana.crt + GF_SERVER_CERT_KEY: /etc/grafana/certs/grafana.key + GF_SERVER_HTTP_PORT: "3000" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_SECURITY_ADMIN_USER: admin + env_file: + - secrets/grafana.env + ports: + - "127.0.0.1:3001:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./secrets/pki/ca.crt:/etc/grafana/certs/ca.crt:ro + - ./secrets/pki/grafana.crt:/etc/grafana/certs/grafana.crt:ro + - ./secrets/pki/grafana.key:/etc/grafana/certs/grafana.key:ro + mem_limit: 192m + cpus: "0.25" + pids_limit: 64 + networks: [telemetry] + restart: unless-stopped + + probe: + profiles: ["tools"] + image: mtusi-lab7-messenger:1.7.0 + build: . + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + entrypoint: ["python", "-m", "app.probes.security"] + <<: *hardening + volumes: + - ./secrets/pki/ca.crt:/probe/ca.crt:ro + - ./secrets/pki/lb.crt:/probe/lb.crt:ro + - ./secrets/pki/lb.key:/probe/lb.key:ro + - ./secrets/pki/api-client.crt:/probe/api.crt:ro + - ./secrets/pki/api-client.key:/probe/api.key:ro + - ./secrets/pki/worker-client.crt:/probe/worker.crt:ro + - ./secrets/pki/worker-client.key:/probe/worker.key:ro + - ./secrets/pki/stranger.crt:/probe/stranger.crt:ro + - ./secrets/pki/stranger.key:/probe/stranger.key:ro + - ./secrets/pki/other-ca.crt:/probe/other-ca.crt:ro + - ./secrets/pki/other-client.crt:/probe/other.crt:ro + - ./secrets/pki/other-client.key:/probe/other.key:ro + - ./secrets/pki/expired.crt:/probe/expired.crt:ro + - ./secrets/pki/expired.key:/probe/expired.key:ro + - ./tests/fixtures/sample.png:/probe/sample.png:ro + networks: [app, telemetry] + restart: "no" + +networks: + app: + telemetry: + +volumes: + pg-data: + s3-data: + rabbit-data: + valkey-data: + bao-data: + prometheus-data: + loki-data: + tempo-data: + grafana-data: diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..8951914 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,33 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +CA стенда выпускается локально (`python scripts/bootstrap.py`) и не коммитится. Проверка ниже смотрит и цепочку, и имя. Доверенный CA сам по себе не делает сертификат балансировщиком: API дополнительно требует SAN `lb.messenger.internal` и serial из `secrets/pki/lb-allowed-serials.txt`. | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | - -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +| Клиент → ingress/LB | TLS 1.2+, CA + SAN `localhost` / `127.0.0.1`, Bearer после входа. Публичный порт только `127.0.0.1:8443` | Сессия истекает по `SESSION_TTL_SECONDS`; logout ставит `revoked_at`. Вход ограничен общим для реплик счётчиком Valkey, `429` и `Retry-After` | `make test CA_FILE=secrets/pki/ca.crt`, `make test-security`, probe `login-rate-limit` | +| LB → API (обе реплики) | mTLS. Сертификат сервера содержит SAN `messenger_api`. Клиент LB обязан иметь SAN `lb.messenger.internal` и разрешённый serial. Порт `8080` слушает `127.0.0.1` и из сети `app` не открывается | Смена leaf LB — замена файлов и allowlist без пересборки образа. `proxy_ssl_session_reuse off`, поэтому новое соединение видит отзыв | `app/mtls_proxy.py`, `infra/nginx/ingress.conf`, probe `lb-identity-allowed`, `worker-cert-rejected-by-api`, `api-without-client-cert`, `api-foreign-ca` | +| API/worker → PostgreSQL | `sslmode=verify-full`, SAN `postgres`, `hostnossl` отвергается. Роли `api_login` и `worker_login` без superuser | Пароли ролей лежат в OpenBao KV и ротируются записью в KV плюс `ALTER ROLE` без пересборки. Worker не имеет `SELECT` на `users` | `infra/postgres/pg_hba.conf`, `03-grants.sql`, track1 `db_select_password_hash=deny` | +| API/worker → S3 | HTTPS SeaweedFS (`minio:8333`), проверка CA, path-style, plaintext-порт S3 выключен. Разные ключи: API пишет только `staging/*` и читает `originals/*`,`variants/*`; worker пишет результаты и не читает `canary/*` | Идентичности генерируются в `secrets/seaweed/s3.json` и не коммитятся. Префиксы проверяются треком 1 | `scripts/bootstrap.py`, track1 | +| API/worker → broker | RabbitMQ слушает только `5671`. `verify_peer`, client cert обязателен, механизм EXTERNAL, пользователь равен CN. Плагин `rabbitmq_auth_mechanism_ssl` включён | API публикует только в exchange `attachment.process` (write на это имя). Worker читает очередь `attachment.process.v1` и не может публиковать в default exchange. Чужой CN того же CA не имеет пользователя | `infra/rabbitmq/rabbitmq.conf`, probe `api-cannot-consume`, `worker-cannot-publish`, `stranger-broker-rejected` | +| API/worker → Collector | Прямого маршрута к `otel-collector` из сети `app` нет. `otel-edge:4318` требует mTLS и CN `api` или `worker`, плюс serial из `secrets/nginx/serials.conf` | Stranger, подписанный тем же CA, получает 403. API и worker предъявляют разные leaf | `infra/nginx/otel-edge.conf`, probe `stranger-collector-rejected` | +| Collector → telemetry backends | Collector проверяет серверные сертификаты Prometheus (`:8889`), Loki (`:3100`) и Tempo (`:4317`). `insecure: true` нет. Сервисы только в сети `telemetry` | Plaintext на Loki не разбирается как HTTP. Grafana ходит по HTTPS со своим CA | `infra/otel/collector.yaml`, probe `loki-plaintext-rejected` | +| Grafana → data sources | Grafana HTTPS на `127.0.0.1:3001` (контейнерный порт 3000). Datasource'ы с `tlsAuthWithCACert` и `serverName`, CA из файла, не `tlsSkipVerify` | Учётная запись admin задаётся локальным `secrets/grafana.env` | `infra/grafana/provisioning/datasources/datasources.yml` | +| API/worker → OpenBao | Listener требует client cert нашего CA. Логин — `auth/cert` leaf'ом роли. Policy читает только `secret/data//*`, запись и `sys/` запрещены. Token TTL 20m | Renewal каждые 15 с, при ошибке процесс логинится заново leaf-сертификатом. Root token в контейнеры API/worker не монтируется | `infra/openbao/config.hcl`, `policies/*.hcl`, `app/secrets_loader.py`, probe OpenBao | +| Оператор → OpenBao / Grafana / RabbitMQ | OpenBao и RabbitMQ management не опубликованы. Оператор вызывает `docker compose exec` и `scripts/unseal.py`. Grafana только на loopback | Root token не монтируется в API/worker. Cert `operator` не состоит в auth-роли приложения | `scripts/unseal.py`, отсутствие `BAO_TOKEN` в `compose.yaml` сервисов api/worker | +| API → Valkey | TLS, SAN `valkey`. Клиентский сертификат не обязателен (`tls-auth-clients no`), потому что health-пользователь умеет только `PING`. Пользователь `api` ограничен ключами `login:*` и командами счётчика, включая `INCRBY`. Worker к Valkey не подключён | Общий счётчик для обеих реплик. Пароль приходит из KV `secret/api/valkey` | `infra/valkey/valkey.conf`, probe `login-rate-limit` | +| Replication | N/A. Учебный стенд одноузловой: PostgreSQL, SeaweedFS, RabbitMQ, Valkey и OpenBao без репликации. Отказоустойчивость заявлена только для двух реплик API за LB | N/A | `compose.yaml` | ## Обязательные отрицательные проверки -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. +1. Доверенный CA и `localhost` дают 200. Пустой trust store в `make test-security` даёт `SSLCertVerificationError`, а не таймаут. +2. LB→API: нет сертификата, чужой CA, сертификат worker или stranger при нашем CA — отказ. Leaf LB с разрешённым serial — 200. Соединение на `api-1:8080` из другого контейнера не устанавливается. +3. Broker и Collector повторяют тот же набор. Сетевая изоляция collector не заменяет проверку CN: stranger доходит до `otel-edge` и получает 403. +4. Пользователь вне чата получает 404 на metadata и content. `X-User-Id` не меняет Bearer. Это делает публичный smoke. +5. Policy `api` читает `secret/data/api/db` и не читает `worker`. Запись KV отклоняется. Worker не получает пароль `api_login`. +6. Отзыв serial LB и смена секрета worker проверяются новым соединением (`proxy_ssl_session_reuse off`, рестарт worker). Старый serial и старый S3 secret не принимаются. ## Bootstrap и эксплуатация -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. +В Git только политики, шаблоны и скрипты. `secrets/`, `.env` и `*.key` игнорируются. CA private key, unseal shares и root token создаёт оператор на хосте. Контейнер OpenBao их не монтирует. Поле `disable_mlock` в OpenBao 2.2 не задаётся: эта версия отвергает его в конфигурации. Данные vault лежат на отдельном volume, процесс работает от uid 100. -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +После обычного рестарта API unseal не нужен. После рестарта OpenBao нужен `python scripts/unseal.py`. Пока vault sealed, новый API не становится ready и не переходит на plaintext. diff --git a/docs/runbooks/api-error-budget.md b/docs/runbooks/api-error-budget.md new file mode 100644 index 0000000..ee152e2 --- /dev/null +++ b/docs/runbooks/api-error-budget.md @@ -0,0 +1,18 @@ +# API error budget + +SLO: 99% of business HTTP responses are not 5xx, measured on routes other than `/health/*`. +4xx is a client outcome and is outside the numerator. The ratio is `5xx / all business requests`. + +Windows used on this lab stand: + +- fast burn: 1 minute and 5 minutes, both above 14.4% (14.4 times a 1% budget), `for` 30s, and at least 0.2 requests/s +- slow burn: 5 minutes and 15 minutes, both above 6%, `for` 1 minute +- simple page: 5 minute ratio above 5% for 1 minute + +A short spike can move the 1 minute window without moving the 5 minute window enough, so the fast burn stays inactive. A sustained burst moves both. + +1. Open Prometheus `ALERTS` or the Messenger dashboard panel "Prometheus alert state". +2. If `ApiErrorBudgetFastBurn` or `ApiHighErrorRate` is firing, open a trace for a 5xx request. +3. If the `db.transaction` span is the long or erroring child, follow `docs/runbooks/db-latency.md`. +4. If the server span is long and the database span is short, the handler itself is slow (`handler_slow_ms` fault or application code). +5. Remove the fault file and wait for the next evaluations. The alert leaves `firing` when the expression is false; `for` does not delay resolution. diff --git a/docs/runbooks/db-latency.md b/docs/runbooks/db-latency.md new file mode 100644 index 0000000..64b04ef --- /dev/null +++ b/docs/runbooks/db-latency.md @@ -0,0 +1,15 @@ +# Slow or unavailable database + +`messenger_db_up` is 0 when the process cannot run `SELECT 1`. Business calls then return 503 `unavailable`. That increases the API error ratio. Health `/health/live` stays 200. `/health/ready` becomes 503. + +A slow database is different from a slow handler: + +- slow database: the `db.transaction` span is long and the server span is about the same length +- slow handler: `handler_slow` sits in the server span, and `db.transaction` stays short + +`DatabaseUnready` fires when `min(messenger_db_up) == 0` for 30 seconds. + +1. Compare one server span with its `db.transaction` child in Tempo. +2. If the database span failed, check PostgreSQL. Grafana being down does not cause this alert. +3. Restore PostgreSQL. `messenger_db_up` returns to 1 and the error-budget alerts resolve after the windows slide. +4. The collector and Grafana are not on the readiness path. diff --git a/docs/runbooks/queue-lag.md b/docs/runbooks/queue-lag.md new file mode 100644 index 0000000..406e451 --- /dev/null +++ b/docs/runbooks/queue-lag.md @@ -0,0 +1,14 @@ +# Queue lag and worker loss + +`messenger_queue_oldest_age_seconds` is the age of the oldest attachment still `queued` or `processing`. It is not the duration of a completed job. `messenger_queue_depth` is how many such rows exist. Both are read from PostgreSQL by the API, so they keep moving when the worker process is stopped. + +`messenger_worker_consumers` is the RabbitMQ consumer count. The API polls the management API, so the series becomes 0 instead of going stale. + +Alert `QueueLagHigh`: max age above 30 seconds for 1 minute. +Alert `WorkerConsumersZero`: max consumers equals 0 for 1 minute. + +1. Confirm consumers are 0 or age is climbing on the Messenger dashboard. +2. Check worker logs in Loki for `job.failed` or `broker.reconnect`. A clean stop has neither new `job.consumed` nor a database error. +3. Start the worker again: `docker compose --env-file .env start worker`. +4. Age returns to 0 after the backlog is acknowledged. Alerts resolve on the next evaluation. +5. Do not use p95 time-to-ready alone: it only observes jobs that reached `ready`. diff --git a/docs/runbooks/telemetry-backend.md b/docs/runbooks/telemetry-backend.md new file mode 100644 index 0000000..fb6559e --- /dev/null +++ b/docs/runbooks/telemetry-backend.md @@ -0,0 +1,8 @@ +# Telemetry backend unavailable + +The API and worker export OTLP on a background queue (256 spans or log records, 5 second timeout). The collector limits itself to 192 MiB, batches, and keeps a sending queue of 200 items with retry capped at 30 seconds. When Tempo, Loki, or Prometheus is down, exports fail and `messenger_telemetry_dropped_total` increases. Message acceptance does not wait on those exports. + +1. Stop one backend, for example `docker compose --env-file .env stop tempo`. +2. Send a few API requests. `/health/ready` stays 200 and messages still return 201. +3. Watch collector memory. It should stay near its limit, not grow without bound. +4. Start the backend again. New spans appear. Spans dropped during the outage are not replayed from the in-memory queue once it has overflowed. diff --git a/docs/runbooks/time-to-ready.md b/docs/runbooks/time-to-ready.md new file mode 100644 index 0000000..833e5f3 --- /dev/null +++ b/docs/runbooks/time-to-ready.md @@ -0,0 +1,11 @@ +# Time to ready + +SLO: 95% of attachments that reach `ready` do so within 10 seconds of durable acceptance. +The series is `messenger_attachment_time_to_ready_seconds`, a histogram in seconds. +`AttachmentReadyLatencyHigh` uses `histogram_quantile(0.95, ...)` over 15 minutes and pages after 2 minutes above 10 seconds. + +This SLO does not see jobs that never finish. Pair it with queue age. + +1. If p95 is high and queue age is low, open a worker trace and compare `s3.put`, `s3.get`, and `db.transaction`. +2. A retry span with `attempt > 1` and event `retry` means the first attempt was transient. The successful attempt should still contain S3 and the ready update. +3. If p95 is quiet but users wait, read queue age. The worker is likely stopped or stuck before completion. diff --git a/evidence/argon2-budget.json b/evidence/argon2-budget.json new file mode 100644 index 0000000..cff500d --- /dev/null +++ b/evidence/argon2-budget.json @@ -0,0 +1,12 @@ +{ + "algorithm": "argon2id", + "memory_kib": 19456, + "iterations": 2, + "parallelism": 1, + "hash_ms": 29, + "verify_ms": 25, + "login_limit_per_user_per_minute": 10, + "login_limit_global_per_minute": 60, + "verify_cpu_seconds_per_minute_at_global_cap": 1.5, + "note": "Rate limit runs before Argon2id. Parameters are the required floor; measured verify is about 25 ms on this stand." +} diff --git a/evidence/track1-least-privilege.json b/evidence/track1-least-privilege.json new file mode 100644 index 0000000..7e7f717 --- /dev/null +++ b/evidence/track1-least-privilege.json @@ -0,0 +1,12 @@ +{ + "track": 1, + "identity": "worker.messenger.internal", + "db_select_attachment_metadata": "allow", + "db_select_password_hash": "deny:42501", + "db_insert_user": "deny:42501", + "db_delete_message": "deny:42501", + "s3_put_original": "allow", + "s3_get_canary": "deny:AccessDenied", + "s3_put_canary": "deny:AccessDenied", + "queue_publish": "deny:403" +} diff --git a/evidence/track1-run.json b/evidence/track1-run.json new file mode 100644 index 0000000..7643c2c --- /dev/null +++ b/evidence/track1-run.json @@ -0,0 +1,41 @@ +{ + "least_privilege_exit": 0, + "least_privilege": { + "db_delete_message": "deny:42501", + "db_insert_user": "deny:42501", + "db_select_attachment_metadata": "allow", + "db_select_password_hash": "deny:42501", + "queue_publish": "deny:403", + "s3_get_canary": "deny:AccessDenied", + "s3_put_canary": "deny:AccessDenied", + "s3_put_original": "allow" + }, + "broad_exit": 1, + "broad": { + "db_delete_message": "deny:42501", + "db_insert_user": "deny:42501", + "db_select_attachment_metadata": "allow", + "db_select_password_hash": "allow", + "queue_publish": "deny:403", + "s3_get_canary": "allow", + "s3_put_canary": "allow", + "s3_put_original": "allow" + }, + "restored_exit": 0, + "restored": { + "db_delete_message": "deny:42501", + "db_insert_user": "deny:42501", + "db_select_attachment_metadata": "allow", + "db_select_password_hash": "deny:42501", + "queue_publish": "deny:403", + "s3_get_canary": "deny:AccessDenied", + "s3_put_canary": "deny:AccessDenied", + "s3_put_original": "allow" + }, + "queued_before_revoke": "queued", + "queue_before_revoke": 1, + "queue_after_revoke": 1, + "old_secret": "deny:SignatureDoesNotMatch", + "after_restore": "ready", + "attachment_id_prefix": "3a8962ba" +} \ No newline at end of file diff --git a/evidence/verify-local.json b/evidence/verify-local.json new file mode 100644 index 0000000..6a46164 --- /dev/null +++ b/evidence/verify-local.json @@ -0,0 +1,27 @@ +{ + "rehash": { + "login": 201, + "params_before": "m=8192,t=1,p=1", + "params_after": "m=19456,t=2,p=1", + "expired_session": 401 + }, + "leaf": { + "new_serial_ready": "200", + "old_serial": "http 403", + "public_after_reload": 200 + }, + "telemetry_leak_counts": { + "Bearer ": 0, + "password=": 0, + "postgresql://": 0, + "aws_secret_access_key": 0, + "BEGIN PRIVATE": 0 + }, + "sealed": { + "ready_before": 200, + "login_before_seal": 401, + "login_while_sealed": 503, + "secret_env": [] + }, + "ready_after_unseal": 200 +} \ No newline at end of file diff --git a/faults/README.md b/faults/README.md new file mode 100644 index 0000000..d78829e --- /dev/null +++ b/faults/README.md @@ -0,0 +1,11 @@ +Local fault-injection files. They are not present by default, so the API behaves normally. + +Write one of these names in this directory while the stack is running: + +- `http_error_ratio` — float 0..1, fraction of `/api/` requests that return 500 +- `db_slow_ms` — delay added inside the `db.transaction` span +- `db_fail` — `1` makes database calls return 503 +- `handler_slow_ms` — delay inside the HTTP server span and outside the database span +- `process_fail_remaining` — positive integer, worker fails that many processing attempts once each + +Do not put secrets here. The worker must be able to update `process_fail_remaining`. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..e10a727 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,187 @@ +{ + "uid": "messenger-overview", + "title": "Messenger observability", + "timezone": "utc", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "tags": ["messenger", "lab6"], + "time": {"from": "now-1h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "text", + "title": "Signal stores", + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 0}, + "options": { + "mode": "markdown", + "content": "Prometheus stores metrics (rates, histograms, gauges). Loki stores structured logs. Tempo stores traces. The collector is not a database: it receives OTLP, redacts, batches, and exports. Grafana only queries those stores. Percentiles use `histogram_quantile` on cumulative buckets, summed across API replicas before the quantile." + } + }, + { + "id": 2, + "type": "timeseries", + "title": "API RPS by route", + "description": "Unit: requests per second. Sum of rates across replicas.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4}, + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []}, + "targets": [ + { + "refId": "A", + "expr": "sum by (http_route) (rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))", + "legendFormat": "{{http_route}}" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "API error ratio (5xx / business)", + "description": "Unit: fraction 0-1. Numerator is status class 5xx. Denominator is business routes, excluding /health and unmatched. 4xx is not an error for this SLI.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4}, + "fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "messenger:api_error_ratio:rate5m", "legendFormat": "5m"}, + {"refId": "B", "expr": "messenger:api_error_ratio:rate1m", "legendFormat": "1m"} + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "API duration p50/p95/p99", + "description": "Unit: seconds. histogram_quantile over the sum of bucket rates. Not an average of per-replica quantiles.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p50"}, + {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p95"}, + {"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p99"} + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Jobs by outcome", + "description": "Unit: jobs per second. outcome is success, failed, or retry. job_type is attachment.process.v1.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12}, + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", "legendFormat": "{{outcome}}"} + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Worker job duration p95", + "description": "Unit: seconds, from the job histogram.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 20}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"} + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth and oldest age", + "description": "Depth is a count of queued+processing attachments. Age is seconds since the oldest of those rows was accepted. max() avoids double-counting the two API replicas.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 20}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "age"}, "properties": [{"id": "unit", "value": "s"}]}]}, + "targets": [ + {"refId": "A", "expr": "max(messenger_queue_depth{queue=\"attachment.process.v1\"})", "legendFormat": "depth"}, + {"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachment.process.v1\"})", "legendFormat": "age"} + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Time to ready p95", + "description": "Unit: seconds from durable acceptance to ready. Completed jobs only.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 20}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m])))", "legendFormat": "p95"} + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "DB pool used and waiting", + "description": "Unit: connections. Summed across processes. waiting is requests blocked on the pool.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "sum(messenger_db_pool_used)", "legendFormat": "used"}, + {"refId": "B", "expr": "sum(messenger_db_pool_waiting)", "legendFormat": "waiting"} + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker saturation and consumers", + "description": "busy/capacity are counts inside the worker. consumers is the RabbitMQ consumer count observed by the API.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"}, + {"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"}, + {"refId": "C", "expr": "max(messenger_worker_consumers{service_name=\"messenger-api\"})", "legendFormat": "consumers"} + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Process CPU and memory", + "description": "CPU unit is process CPU-seconds per wall second (can exceed 1). Memory unit is bytes of RSS.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 28}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "max by (service_name) (messenger_process_cpu_utilization)", "legendFormat": "cpu {{service_name}}"}, + {"refId": "B", "expr": "max by (service_name) (messenger_process_memory_bytes)", "legendFormat": "rss {{service_name}}"} + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "DB duration p95", + "description": "Unit: seconds. Includes an injected db_slow fault. Compare with the server span to separate a slow database from a slow handler.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [ + {"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_db_duration_seconds_bucket[5m])))", "legendFormat": "db p95"} + ] + }, + { + "id": 13, + "type": "table", + "title": "Prometheus alert state", + "description": "Local firing/pending state. No external notification channel is configured.", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36}, + "targets": [ + {"refId": "A", "expr": "ALERTS{alertstate=~\"firing|pending\"}", "instant": true, "format": "table"} + ] + }, + { + "id": 14, + "type": "logs", + "title": "API and worker logs", + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 44}, + "targets": [ + {"refId": "A", "expr": "{service_name=~\"messenger-api|messenger-worker\"}"} + ] + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..3ed9466 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yml b/infra/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..6a6e363 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,53 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 5s + tlsAuthWithCACert: true + serverName: prometheus + secureJsonData: + tlsCACert: $__file{/etc/grafana/certs/ca.crt} + - name: Loki + uid: loki + type: loki + access: proxy + url: https://loki:3100 + editable: false + jsonData: + tlsAuthWithCACert: true + serverName: loki + derivedFields: + - name: TraceID + matcherRegex: "\"trace_id\":\"([0-9a-f]+)\"" + url: "$${__value.raw}" + datasourceUid: tempo + urlDisplayLabel: View trace + secureJsonData: + tlsCACert: $__file{/etc/grafana/certs/ca.crt} + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tempo:3200 + editable: false + jsonData: + tlsAuthWithCACert: true + serverName: tempo + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: loki + secureJsonData: + tlsCACert: $__file{/etc/grafana/certs/ca.crt} diff --git a/infra/loki/loki.yml b/infra/loki/loki.yml new file mode 100644 index 0000000..d2a442c --- /dev/null +++ b/infra/loki/loki.yml @@ -0,0 +1,44 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + log_level: warn + http_tls_config: + cert_file: /certs/loki.crt + key_file: /certs/loki.key + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + volume_enabled: true + reject_old_samples: false + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_query_series: 500 + +analytics: + reporting_enabled: false diff --git a/infra/minio/api-s3.json b/infra/minio/api-s3.json new file mode 100644 index 0000000..1567c48 --- /dev/null +++ b/infra/minio/api-s3.json @@ -0,0 +1,24 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"], + "Condition": { "StringLike": { "s3:prefix": ["staging/*", "originals/*", "variants/*"] } } + } + ] +} diff --git a/infra/minio/worker-broad.json b/infra/minio/worker-broad.json new file mode 100644 index 0000000..44afff6 --- /dev/null +++ b/infra/minio/worker-broad.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["arn:aws:s3:::messenger-attachments", "arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/minio/worker-s3.json b/infra/minio/worker-s3.json new file mode 100644 index 0000000..de3b86f --- /dev/null +++ b/infra/minio/worker-s3.json @@ -0,0 +1,24 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": [ + "arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/variants/*" + ] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"], + "Condition": { "StringLike": { "s3:prefix": ["staging/*", "originals/*", "variants/*"] } } + } + ] +} diff --git a/infra/nginx/ingress.conf b/infra/nginx/ingress.conf new file mode 100644 index 0000000..bf39f04 --- /dev/null +++ b/infra/nginx/ingress.conf @@ -0,0 +1,34 @@ +upstream messenger_api { + server api-1:8443 max_fails=2 fail_timeout=5s; + server api-2:8443 max_fails=2 fail_timeout=5s; +} + +server { + listen 8443 ssl; + server_name localhost; + + ssl_certificate /certs/ingress.crt; + ssl_certificate_key /certs/ingress.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + + proxy_connect_timeout 2s; + proxy_send_timeout 30s; + proxy_read_timeout 60s; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + proxy_next_upstream_tries 2; + + location / { + proxy_pass https://messenger_api; + proxy_ssl_certificate /certs/lb.crt; + proxy_ssl_certificate_key /certs/lb.key; + proxy_ssl_trusted_certificate /certs/ca.crt; + proxy_ssl_verify on; + proxy_ssl_name messenger_api; + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + proxy_ssl_session_reuse off; + proxy_set_header Host $host; + proxy_set_header Connection ""; + } +} diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 0000000..f6c192c --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,15 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /tmp/nginx.pid; + +events { + worker_connections 256; +} + +http { + access_log off; + map_hash_bucket_size 128; + client_max_body_size 11m; + proxy_http_version 1.1; + include /etc/nginx/conf.d/*.conf; +} diff --git a/infra/nginx/otel-edge.conf b/infra/nginx/otel-edge.conf new file mode 100644 index 0000000..2cee5da --- /dev/null +++ b/infra/nginx/otel-edge.conf @@ -0,0 +1,26 @@ +include /etc/nginx/serials.conf; + +server { + listen 4318 ssl; + server_name otel-edge; + + ssl_certificate /certs/server.crt; + ssl_certificate_key /certs/server.key; + ssl_client_certificate /certs/ca.crt; + ssl_verify_client on; + ssl_verify_depth 2; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + if ($otel_cn_ok = 0) { return 403; } + if ($otel_serial_ok = 0) { return 403; } + resolver 127.0.0.11 ipv6=off valid=10s; + set $otel_upstream otel-collector:4318; + proxy_pass http://$otel_upstream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Connection ""; + proxy_read_timeout 10s; + proxy_send_timeout 10s; + } +} diff --git a/infra/openbao/config.hcl b/infra/openbao/config.hcl new file mode 100644 index 0000000..5376737 --- /dev/null +++ b/infra/openbao/config.hcl @@ -0,0 +1,18 @@ +ui = false +log_level = "warn" + +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/openbao/tls/server.crt" + tls_key_file = "/openbao/tls/server.key" + tls_client_ca_file = "/openbao/tls/ca.crt" + tls_require_and_verify_client_cert = true + tls_min_version = "tls12" +} + +storage "file" { + path = "/openbao/data" +} + +api_addr = "https://openbao:8200" +cluster_addr = "https://openbao:8201" diff --git a/infra/openbao/policies/api.hcl b/infra/openbao/policies/api.hcl new file mode 100644 index 0000000..37b492d --- /dev/null +++ b/infra/openbao/policies/api.hcl @@ -0,0 +1,27 @@ +path "secret/data/api/*" { + capabilities = ["read"] +} + +path "secret/metadata/api/*" { + capabilities = ["read", "list"] +} + +path "secret/data/worker/*" { + capabilities = ["deny"] +} + +path "secret/metadata/worker/*" { + capabilities = ["deny"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "sys/*" { + capabilities = ["deny"] +} diff --git a/infra/openbao/policies/worker.hcl b/infra/openbao/policies/worker.hcl new file mode 100644 index 0000000..fb4d3aa --- /dev/null +++ b/infra/openbao/policies/worker.hcl @@ -0,0 +1,27 @@ +path "secret/data/worker/*" { + capabilities = ["read"] +} + +path "secret/metadata/worker/*" { + capabilities = ["read", "list"] +} + +path "secret/data/api/*" { + capabilities = ["deny"] +} + +path "secret/metadata/api/*" { + capabilities = ["deny"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "sys/*" { + capabilities = ["deny"] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..03a7f13 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,103 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 160 + spike_limit_mib: 40 + attributes/redact: + actions: + - key: url.full + action: delete + - key: url.query + action: delete + - key: url.path + action: delete + - key: http.url + action: delete + - key: http.target + action: delete + - key: db.statement + action: delete + - key: db.query.text + action: delete + - key: http.request.header.authorization + action: delete + - key: http.request.header.cookie + action: delete + batch: + timeout: 2s + send_batch_size: 256 + send_batch_max_size: 512 + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: false + ca_file: /certs/ca.crt + cert_file: /certs/collector.crt + key_file: /certs/collector.key + server_name_override: tempo + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + otlphttp/loki: + logs_endpoint: https://loki:3100/otlp/v1/logs + tls: + insecure: false + ca_file: /certs/ca.crt + cert_file: /certs/collector.crt + key_file: /certs/collector.key + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 15s + max_elapsed_time: 30s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + prometheus: + endpoint: 0.0.0.0:8889 + tls: + cert_file: /certs/collector.crt + key_file: /certs/collector.key + resource_to_telemetry_conversion: + enabled: true + metric_expiration: 180s + +service: + extensions: [health_check] + telemetry: + metrics: + address: 127.0.0.1:8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/redact, batch] + exporters: [otlphttp/loki] diff --git a/infra/postgres/03-grants.sql b/infra/postgres/03-grants.sql new file mode 100644 index 0000000..b43a847 --- /dev/null +++ b/infra/postgres/03-grants.sql @@ -0,0 +1,13 @@ +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO api_login, worker_login; + +GRANT SELECT, INSERT, UPDATE ON + users, sessions, chats, chat_members, messages, + attachments, attachment_variants, message_attachments, + idempotency, outbox +TO api_login; + +GRANT SELECT, UPDATE ON attachments TO worker_login; +GRANT SELECT, INSERT, UPDATE ON attachment_variants TO worker_login; + +REVOKE ALL ON users, sessions, chats, chat_members, messages, idempotency, outbox FROM worker_login; diff --git a/infra/postgres/Dockerfile b/infra/postgres/Dockerfile new file mode 100644 index 0000000..2d4f9fc --- /dev/null +++ b/infra/postgres/Dockerfile @@ -0,0 +1,7 @@ +FROM postgres:16.6-alpine + +COPY entrypoint.sh /usr/local/bin/lab-entrypoint.sh +RUN chmod 755 /usr/local/bin/lab-entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/lab-entrypoint.sh"] +CMD ["postgres"] diff --git a/infra/postgres/entrypoint.sh b/infra/postgres/entrypoint.sh new file mode 100644 index 0000000..d73bc00 --- /dev/null +++ b/infra/postgres/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p /var/lib/postgresql/certs +cp /certs-src/server.crt /var/lib/postgresql/certs/server.crt +cp /certs-src/server.key /var/lib/postgresql/certs/server.key +cp /certs-src/ca.crt /var/lib/postgresql/certs/ca.crt +chown postgres:postgres /var/lib/postgresql/certs/server.crt /var/lib/postgresql/certs/server.key /var/lib/postgresql/certs/ca.crt +chmod 600 /var/lib/postgresql/certs/server.key +exec docker-entrypoint.sh postgres \ + -c ssl=on \ + -c ssl_cert_file=/var/lib/postgresql/certs/server.crt \ + -c ssl_key_file=/var/lib/postgresql/certs/server.key \ + -c ssl_ca_file=/var/lib/postgresql/certs/ca.crt \ + -c ssl_min_protocol_version=TLSv1.2 \ + -c hba_file=/etc/postgres/pg_hba.conf \ + -c password_encryption=scram-sha-256 diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..b6e6cd1 --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,3 @@ +local all all trust +hostssl all all all scram-sha-256 +hostnossl all all all reject diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..cd6e894 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_timeout: 4s + +rule_files: + - /etc/prometheus/rules.yml + +scrape_configs: + - job_name: otel-collector + scheme: https + tls_config: + ca_file: /etc/prometheus/ca.crt + server_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml new file mode 100644 index 0000000..daddf41 --- /dev/null +++ b/infra/prometheus/rules.yml @@ -0,0 +1,96 @@ +groups: + - name: messenger-sli + interval: 5s + rules: + - record: messenger:api_requests:rate1m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m])) + - record: messenger:api_5xx:rate1m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m])) + - record: messenger:api_error_ratio:rate1m + expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9) + - record: messenger:api_requests:rate5m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m])) + - record: messenger:api_5xx:rate5m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m])) + - record: messenger:api_error_ratio:rate5m + expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9) + - record: messenger:api_requests:rate15m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m])) + - record: messenger:api_5xx:rate15m + expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m])) + - record: messenger:api_error_ratio:rate15m + expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9) + + - name: messenger-alerts + interval: 5s + rules: + - alert: ApiHighErrorRate + expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2 + for: 1m + labels: + severity: page + slo: api-availability + annotations: + summary: API business 5xx ratio is above 5% + description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m. + runbook: docs/runbooks/api-error-budget.md + - alert: ApiErrorBudgetFastBurn + expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2 + for: 30s + labels: + severity: page + slo: api-availability + burn: fast + annotations: + summary: Fast multi-window burn of the 1% API error budget + description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present. + runbook: docs/runbooks/api-error-budget.md + - alert: ApiErrorBudgetSlowBurn + expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2 + for: 1m + labels: + severity: ticket + slo: api-availability + burn: slow + annotations: + summary: Slow multi-window burn of the API error budget + description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss. + runbook: docs/runbooks/api-error-budget.md + - alert: QueueLagHigh + expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30 + for: 1m + labels: + severity: page + slo: time-to-ready + annotations: + summary: Oldest attachment job is older than 30s + description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds. + runbook: docs/runbooks/queue-lag.md + - alert: WorkerConsumersZero + expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0 + for: 1m + labels: + severity: page + annotations: + summary: RabbitMQ has no attachment consumers + description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops. + runbook: docs/runbooks/queue-lag.md + - alert: AttachmentReadyLatencyHigh + expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10 + for: 2m + labels: + severity: ticket + slo: time-to-ready + annotations: + summary: p95 time to ready is above 10s + description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that. + runbook: docs/runbooks/time-to-ready.md + - alert: DatabaseUnready + expr: min(messenger_db_up) == 0 + for: 30s + labels: + severity: page + annotations: + summary: An API or worker process cannot use PostgreSQL + description: Distinguish this from a slow handler by comparing the db.transaction span with the server span. + runbook: docs/runbooks/db-latency.md diff --git a/infra/prometheus/web.yml b/infra/prometheus/web.yml new file mode 100644 index 0000000..767e195 --- /dev/null +++ b/infra/prometheus/web.yml @@ -0,0 +1,4 @@ +tls_server_config: + cert_file: /certs/prometheus.crt + key_file: /certs/prometheus.key + min_version: TLS12 diff --git a/infra/rabbitmq/Dockerfile b/infra/rabbitmq/Dockerfile new file mode 100644 index 0000000..a3ff61a --- /dev/null +++ b/infra/rabbitmq/Dockerfile @@ -0,0 +1,7 @@ +FROM rabbitmq:3.13.7-management-alpine + +COPY entrypoint.sh /usr/local/bin/lab-entrypoint.sh +RUN chmod 755 /usr/local/bin/lab-entrypoint.sh \ + && rabbitmq-plugins enable --offline rabbitmq_auth_mechanism_ssl + +ENTRYPOINT ["/usr/local/bin/lab-entrypoint.sh"] diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json new file mode 100644 index 0000000..cdef1e7 --- /dev/null +++ b/infra/rabbitmq/definitions.json @@ -0,0 +1,33 @@ +{ + "vhosts": [{ "name": "/" }], + "exchanges": [ + { + "name": "attachment.process", + "vhost": "/", + "type": "direct", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "queues": [ + { + "name": "attachment.process.v1", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "attachment.process", + "vhost": "/", + "destination": "attachment.process.v1", + "destination_type": "queue", + "routing_key": "attachment.process.v1", + "arguments": {} + } + ] +} diff --git a/infra/rabbitmq/entrypoint.sh b/infra/rabbitmq/entrypoint.sh new file mode 100644 index 0000000..66e082b --- /dev/null +++ b/infra/rabbitmq/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mkdir -p /etc/rabbitmq/certs +cp /certs-src/ca.crt /etc/rabbitmq/certs/ca.crt +cp /certs-src/server.crt /etc/rabbitmq/certs/server.crt +cp /certs-src/server.key /etc/rabbitmq/certs/server.key +chown -R rabbitmq:rabbitmq /etc/rabbitmq/certs +chmod 640 /etc/rabbitmq/certs/server.key +exec docker-entrypoint.sh rabbitmq-server diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..fdab95f --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,17 @@ +listeners.tcp = none +listeners.ssl.default = 5671 +management.tcp.port = 15672 +management.tcp.ip = 127.0.0.1 +prometheus.tcp.port = 15692 +prometheus.tcp.ip = 127.0.0.1 +ssl_options.cacertfile = /etc/rabbitmq/certs/ca.crt +ssl_options.certfile = /etc/rabbitmq/certs/server.crt +ssl_options.keyfile = /etc/rabbitmq/certs/server.key +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true +ssl_options.versions.1 = tlsv1.2 +ssl_options.versions.2 = tlsv1.3 +auth_mechanisms.1 = EXTERNAL +ssl_cert_login_from = common_name +management.load_definitions = /etc/rabbitmq/definitions.json +log.console.level = warning diff --git a/infra/tempo/tempo.yml b/infra/tempo/tempo.yml new file mode 100644 index 0000000..8c283c7 --- /dev/null +++ b/infra/tempo/tempo.yml @@ -0,0 +1,41 @@ +stream_over_http_enabled: true + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + log_level: warn + http_tls_config: + cert_file: /certs/tempo.crt + key_file: /certs/tempo.key + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: /certs/tempo.crt + key_file: /certs/tempo.key + min_version: "1.2" + +ingester: + trace_idle_period: 10s + max_block_duration: 5m + +compactor: + compaction: + block_retention: 48h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +querier: + max_concurrent_queries: 4 diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 0000000..2b1c461 --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,14 @@ +bind 0.0.0.0 +port 0 +tls-port 6379 +tls-cert-file /certs/server.crt +tls-key-file /certs/server.key +tls-ca-cert-file /certs/ca.crt +tls-auth-clients no +tls-protocols "TLSv1.2 TLSv1.3" +aclfile /etc/valkey/users.acl +dir /data +appendonly yes +appendfsync everysec +protected-mode no +save "" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b3cc1d2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +starlette==0.45.3 +uvicorn==0.34.0 +psycopg[binary,pool]==3.2.6 +argon2-cffi==23.1.0 +boto3==1.37.38 +pika==1.3.2 +Pillow==11.1.0 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +redis==5.2.1 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..dbeb96a --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,544 @@ +"""Create local CA, secrets and, with --up, initialize the stand. Never prints secret values.""" +import datetime +import ipaddress +import json +import os +import secrets +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PKI = ROOT / "secrets" / "pki" +OPERATOR = ROOT / "secrets" / "operator" + + +def ensure_cryptography(): + try: + import cryptography # noqa: F401 + return + except ImportError: + pass + venv = ROOT / ".venv-bootstrap" + py = venv / "Scripts" / "python.exe" + if not py.exists(): + subprocess.check_call([sys.executable, "-m", "venv", str(venv)]) + subprocess.check_call([str(py), "-m", "pip", "install", "cryptography==44.0.2"]) + os.execv(str(py), [str(py), str(Path(__file__).resolve()), *sys.argv[1:]]) + + +def write_pem(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + +def build_material(): + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + def private(): + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + def dump_key(key): + return key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + + def dump_cert(cert): + return cert.public_bytes(serialization.Encoding.PEM) + + def name(cn): + return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]) + + now = datetime.datetime.now(datetime.timezone.utc) + + def certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired=False): + key = private() + if expired: + start = now - datetime.timedelta(days=3) + end = now - datetime.timedelta(days=1) + else: + start = now - datetime.timedelta(minutes=10) + end = now + datetime.timedelta(days=days) + alt = [x509.DNSName(item) for item in sans] + alt += [x509.IPAddress(ipaddress.ip_address(item)) for item in ips] + uses = [] + if "client" in eku: + uses.append(ExtendedKeyUsageOID.CLIENT_AUTH) + if "server" in eku: + uses.append(ExtendedKeyUsageOID.SERVER_AUTH) + builder = ( + x509.CertificateBuilder() + .subject_name(name(cn)) + .issuer_name(issuer_cert.subject if issuer_cert is not None else name(cn)) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(start) + .not_valid_after(end) + .add_extension(x509.BasicConstraints(ca=issuer_cert is None, path_length=0 if issuer_cert is None else None), critical=True) + ) + if issuer_cert is None: + builder = builder.add_extension( + x509.KeyUsage( + digital_signature=True, + key_cert_sign=True, + crl_sign=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + else: + builder = builder.add_extension(x509.SubjectAlternativeName(alt), critical=False) + builder = builder.add_extension(x509.ExtendedKeyUsage(uses), critical=False) + signed = builder.sign(issuer_key if issuer_cert is not None else key, hashes.SHA256()) + return key, signed + + PKI.mkdir(parents=True, exist_ok=True) + if not (PKI / "ca.crt").exists(): + ca_key, ca_cert = certificate("messenger-lab-ca", [], [], [], 30, None, None) + other_key, other_cert = certificate("foreign-lab-ca", [], [], [], 30, None, None) + write_pem(PKI / "ca.key", dump_key(ca_key)) + write_pem(PKI / "ca.crt", dump_cert(ca_cert)) + write_pem(PKI / "other-ca.key", dump_key(other_key)) + write_pem(PKI / "other-ca.crt", dump_cert(other_cert)) + leaves = { + "ingress": ("localhost", ["localhost", "nginx"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), + "lb": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "api-server": ("api.messenger.internal", ["messenger_api", "api-1", "api-2", "api.messenger.internal"], [], ["server"], 14, False, ca_key, ca_cert), + "api-client": ("api.messenger.internal", ["api.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "worker-client": ("worker.messenger.internal", ["worker.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "stranger": ("stranger.messenger.internal", ["stranger.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "expired": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 1, True, ca_key, ca_cert), + "other-client": ("lb.messenger.internal", ["lb.messenger.internal"], [], ["client"], 14, False, other_key, other_cert), + "postgres": ("postgres", ["postgres"], [], ["server"], 14, False, ca_key, ca_cert), + "minio": ("minio", ["minio"], [], ["server"], 14, False, ca_key, ca_cert), + "rabbitmq": ("rabbitmq", ["rabbitmq"], [], ["server"], 14, False, ca_key, ca_cert), + "valkey": ("valkey", ["valkey"], [], ["server", "client"], 14, False, ca_key, ca_cert), + "openbao": ("openbao", ["openbao", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), + "health": ("health.messenger.internal", ["health.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "operator": ("operator.messenger.internal", ["operator.messenger.internal"], [], ["client"], 14, False, ca_key, ca_cert), + "otel-edge": ("otel-edge", ["otel-edge"], [], ["server"], 14, False, ca_key, ca_cert), + "collector": ("otel-collector", ["otel-collector"], [], ["server", "client"], 14, False, ca_key, ca_cert), + "prometheus": ("prometheus", ["prometheus"], [], ["server"], 14, False, ca_key, ca_cert), + "loki": ("loki", ["loki"], [], ["server"], 14, False, ca_key, ca_cert), + "tempo": ("tempo", ["tempo"], [], ["server"], 14, False, ca_key, ca_cert), + "grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], ["server"], 14, False, ca_key, ca_cert), + } + serials = {} + for stem, spec in leaves.items(): + cn, sans, ips, eku, days, expired, issuer_key, issuer_cert = spec + key, cert = certificate(cn, sans, ips, eku, days, issuer_key, issuer_cert, expired) + write_pem(PKI / (stem + ".key"), dump_key(key)) + write_pem(PKI / (stem + ".crt"), dump_cert(cert)) + serials[stem] = cert.serial_number + minio_dir = PKI / "minio" + minio_dir.mkdir(exist_ok=True) + shutil.copyfile(PKI / "minio.crt", minio_dir / "public.crt") + shutil.copyfile(PKI / "minio.key", minio_dir / "private.key") + (PKI / "serials.json").write_text(json.dumps({key: format(value, "X") for key, value in serials.items()}), encoding="utf-8") + return json.loads((PKI / "serials.json").read_text(encoding="utf-8")) + + +def password(): + return secrets.token_hex(24) + + +def material_passwords(): + OPERATOR.mkdir(parents=True, exist_ok=True) + path = OPERATOR / "passwords.json" + current = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + defaults = { + "postgres_password": password(), + "api_db_password": password(), + "worker_db_password": password(), + "minio_root_user": "lab-root", + "minio_root_password": password(), + "api_s3_key": "api-" + secrets.token_hex(6), + "api_s3_secret": password(), + "worker_s3_key": "worker-" + secrets.token_hex(6), + "worker_s3_secret": password(), + "valkey_password": password(), + "grafana_password": password(), + "s3_admin_key": "admin-" + secrets.token_hex(6), + "s3_admin_secret": password(), + } + changed = False + for key, value in defaults.items(): + if key not in current: + current[key] = value + changed = True + if changed: + path.write_text(json.dumps(current), encoding="utf-8") + return current + + +def colon(serial): + hexed = serial if len(serial) % 2 == 0 else "0" + serial + return ":".join(hexed[i : i + 2] for i in range(0, len(hexed), 2)) + + +def write_configs(pw, serials): + (ROOT / "secrets" / "db").mkdir(parents=True, exist_ok=True) + (ROOT / "secrets" / "db" / "01-roles.sql").write_text( + "CREATE ROLE api_login LOGIN PASSWORD '{api}';\nCREATE ROLE worker_login LOGIN PASSWORD '{worker}';\n".format( + api=pw["api_db_password"], worker=pw["worker_db_password"] + ), + encoding="utf-8", + ) + (ROOT / "secrets" / "postgres.env").write_text( + "POSTGRES_USER=messenger\nPOSTGRES_PASSWORD={password}\nPOSTGRES_DB=messenger\n".format(password=pw["postgres_password"]), + encoding="utf-8", + ) + (ROOT / "secrets" / "minio.env").write_text( + "MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\n".format( + user=pw["minio_root_user"], password=pw["minio_root_password"] + ), + encoding="utf-8", + ) + (ROOT / "secrets" / "grafana.env").write_text( + "GF_SECURITY_ADMIN_PASSWORD={password}\n".format(password=pw["grafana_password"]), + encoding="utf-8", + ) + (ROOT / "secrets" / "valkey").mkdir(parents=True, exist_ok=True) + (ROOT / "secrets" / "valkey" / "users.acl").write_text( + "\n".join( + [ + "user default off", + "user health on nopass +ping", + "user api on >{password} ~login:* +incr +incrby +expire +ttl +ping +get".format(password=pw["valkey_password"]), + "", + ] + ), + encoding="utf-8", + ) + (PKI / "lb-allowed-serials.txt").write_text(serials["lb"] + "\n" + colon(serials["lb"]) + "\n", encoding="utf-8") + seen = {} + for stem in ("api-client", "worker-client"): + raw = serials[stem].upper() + for item in (raw, colon(raw)): + seen[item.lower()] = item + lines = ["map $ssl_client_s_dn $otel_cn_ok {", " default 0;", ' "~*CN=api\\.messenger\\.internal" 1;', ' "~*CN=worker\\.messenger\\.internal" 1;', "}"] + lines.append("map $ssl_client_serial $otel_serial_ok {") + lines.append(" default 0;") + for item in seen.values(): + lines.append(' "%s" 1;' % item) + lines.append("}") + nginx = ROOT / "secrets" / "nginx" + nginx.mkdir(parents=True, exist_ok=True) + (nginx / "serials.conf").write_text("\n".join(lines) + "\n", encoding="utf-8") + (ROOT / ".env").write_text( + "\n".join( + [ + "SESSION_TTL_SECONDS=3600", + "IDEMPOTENCY_TTL_SECONDS=86400", + "WORKER_CONCURRENCY=2", + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "STAGING_RETENTION_SECONDS=86400", + "", + ] + ), + encoding="utf-8", + ) + for name in ("api-1", "api-2", "worker"): + (ROOT / "run" / "instances" / name).mkdir(parents=True, exist_ok=True) + + +def compose(*args, check=True, env=None): + command = ["docker", "compose", "--env-file", ".env", *args] + return subprocess.run(command, cwd=ROOT, check=check, env=env) + + +def bao(*args, token=None): + command = [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "-e", + "BAO_ADDR=https://127.0.0.1:8200", + "-e", + "BAO_CACERT=/openbao/tls/ca.crt", + "-e", + "BAO_CLIENT_CERT=/openbao/tls/operator.crt", + "-e", + "BAO_CLIENT_KEY=/openbao/tls/operator.key", + ] + if token: + command += ["-e", "BAO_TOKEN=" + token] + command += ["openbao", "bao", *args] + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + + +def parse_json(text): + start = text.find("{") + end = text.rfind("}") + if start < 0 or end < start: + return None + return json.loads(text[start : end + 1]) + + +def wait_status(): + for _ in range(40): + result = bao("status", "-format=json") + payload = parse_json(result.stdout or "") + if payload is not None and "initialized" in payload: + return payload + time.sleep(2) + raise SystemExit("OpenBao did not start") + + +def init_openbao(pw): + status = wait_status() + init_path = OPERATOR / "init.json" + if not status.get("initialized"): + result = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json") + payload = parse_json(result.stdout or "") + if result.returncode != 0 or payload is None or "root_token" not in payload: + raise SystemExit("OpenBao init failed") + init_path.write_text(json.dumps(payload), encoding="utf-8") + status = wait_status() + if status.get("sealed", True): + keys = json.loads(init_path.read_text(encoding="utf-8"))["unseal_keys_b64"] + for key in keys[:2]: + result = bao("operator", "unseal", key) + if result.returncode != 0: + raise SystemExit("OpenBao unseal failed") + root = json.loads(init_path.read_text(encoding="utf-8"))["root_token"] + bao("secrets", "enable", "-path=secret", "kv-v2", token=root) + bao("policy", "write", "api", "/openbao/policies/api.hcl", token=root) + bao("policy", "write", "worker", "/openbao/policies/worker.hcl", token=root) + bao("auth", "enable", "cert", token=root) + for role, cn in (("api", "api.messenger.internal"), ("worker", "worker.messenger.internal")): + result = bao( + "write", + "auth/cert/certs/" + role, + "display_name=" + role, + "policies=" + role, + "certificate=@/openbao/tls/ca.crt", + "allowed_common_names=" + cn, + "token_ttl=20m", + "token_max_ttl=2h", + token=root, + ) + if result.returncode != 0: + raise SystemExit("OpenBao cert role " + role + " failed") + writes = { + "secret/api/db": ["username=api_login", "password=" + pw["api_db_password"], "host=postgres", "dbname=messenger"], + "secret/worker/db": ["username=worker_login", "password=" + pw["worker_db_password"], "host=postgres", "dbname=messenger"], + "secret/api/s3": ["access_key=" + pw["api_s3_key"], "secret_key=" + pw["api_s3_secret"]], + "secret/worker/s3": ["access_key=" + pw["worker_s3_key"], "secret_key=" + pw["worker_s3_secret"]], + "secret/api/valkey": ["username=api", "password=" + pw["valkey_password"], "host=valkey"], + } + for path, fields in writes.items(): + result = bao("kv", "put", path, *fields, token=root) + if result.returncode != 0: + raise SystemExit("OpenBao kv put failed for " + path.split("/")[1]) + + +def wait_healthy(*services): + for _ in range(50): + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "--format", "{{.Service}} {{.Health}}", *services], + cwd=ROOT, + capture_output=True, + text=True, + ) + rows = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(rows) >= len(services) and all(line.endswith(" healthy") for line in rows): + return + time.sleep(3) + raise SystemExit("services did not become healthy: " + " ".join(services)) + + +def write_s3_config(pw): + bucket = "messenger-attachments" + config = { + "identities": [ + { + "name": "admin", + "credentials": [{"accessKey": pw["s3_admin_key"], "secretKey": pw["s3_admin_secret"]}], + "actions": ["Admin", "Read", "Write", "List"], + }, + { + "name": "api", + "credentials": [{"accessKey": pw["api_s3_key"], "secretKey": pw["api_s3_secret"]}], + "actions": [ + "Read:{0}/staging/*".format(bucket), + "Write:{0}/staging/*".format(bucket), + "List:{0}/staging/*".format(bucket), + "Read:{0}/originals/*".format(bucket), + "List:{0}/originals/*".format(bucket), + "Read:{0}/variants/*".format(bucket), + "List:{0}/variants/*".format(bucket), + ], + }, + { + "name": "worker", + "credentials": [{"accessKey": pw["worker_s3_key"], "secretKey": pw["worker_s3_secret"]}], + "actions": [ + "Read:{0}/staging/*".format(bucket), + "Write:{0}/staging/*".format(bucket), + "List:{0}/staging/*".format(bucket), + "Read:{0}/originals/*".format(bucket), + "Write:{0}/originals/*".format(bucket), + "List:{0}/originals/*".format(bucket), + "Read:{0}/variants/*".format(bucket), + "Write:{0}/variants/*".format(bucket), + "List:{0}/variants/*".format(bucket), + ], + }, + ] + } + target = ROOT / "secrets" / "seaweed" + target.mkdir(parents=True, exist_ok=True) + (target / "s3.json").write_text(json.dumps(config), encoding="utf-8") + + +def configure_bucket(pw): + admin = ROOT / "secrets" / "seaweed" / "admin.json" + admin.write_text( + json.dumps({"access_key": pw["s3_admin_key"], "secret_key": pw["s3_admin_secret"]}), + encoding="utf-8", + ) + result = subprocess.run( + [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7_app", + "-v", + str(PKI / "ca.crt") + ":/certs/ca.crt:ro", + "-v", + str(admin) + ":/certs/admin.json:ro", + "--entrypoint", + "python", + "mtusi-lab7-messenger:1.7.0", + "-c", + ( + "import json,boto3; from botocore.config import Config; " + "admin=json.load(open('/certs/admin.json')); " + "c=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=admin['access_key']," + "aws_secret_access_key=admin['secret_key'],region_name='us-east-1',verify='/certs/ca.crt'," + "config=Config(signature_version='s3v4',s3={'addressing_style':'path'})); " + "buckets=[b['Name'] for b in c.list_buckets().get('Buckets',[])]; " + "c.create_bucket(Bucket='messenger-attachments') if 'messenger-attachments' not in buckets else None; " + "c.put_object(Bucket='messenger-attachments',Key='canary/marker',Body=b'canary-not-a-secret'); " + "print('s3-bootstrap-ok')" + ), + ], + capture_output=True, + text=True, + ) + admin.unlink(missing_ok=True) + if result.returncode != 0 or "s3-bootstrap-ok" not in (result.stdout or ""): + raise SystemExit("s3 bootstrap failed") + + +def configure_rabbit(): + users = { + "api.messenger.internal": ("^attachment\\.process(\\.v1)?$", "^attachment\\.process$", "^$"), + "worker.messenger.internal": ("^attachment\\.process\\.v1$", "^$", "^attachment\\.process\\.v1$"), + } + subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "await_online_nodes", "1"], + cwd=ROOT, + capture_output=True, + text=True, + ) + for user, perms in users.items(): + last = None + for _ in range(10): + subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "add_user", user, password()], + cwd=ROOT, + capture_output=True, + text=True, + ) + last = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "set_permissions", "-p", "/", user, *perms], + cwd=ROOT, + capture_output=True, + text=True, + ) + if last.returncode == 0: + break + time.sleep(3) + if last is None or last.returncode != 0: + raise SystemExit("rabbit permissions failed for a service user") + expressions = [ + 'rabbit_exchange:declare({resource, <<"/">>, exchange, <<"attachment.process">>}, direct, true, false, false, [], <<"bootstrap">>).', + 'rabbit_binding:add({binding, {resource, <<"/">>, exchange, <<"attachment.process">>}, <<"attachment.process.v1">>, {resource, <<"/">>, queue, <<"attachment.process.v1">>}, []}, <<"bootstrap">>).', + ] + for expression in expressions: + topology = subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "eval", expression], + cwd=ROOT, + capture_output=True, + text=True, + ) + if topology.returncode != 0: + raise SystemExit("rabbit topology failed") + subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", "rabbitmqctl", "delete_user", "guest"], + cwd=ROOT, + capture_output=True, + text=True, + ) + + +def wait_ready(): + import ssl + + ctx = ssl.create_default_context(cafile=str(PKI / "ca.crt")) + for _ in range(40): + try: + with urllib.request.urlopen("https://127.0.0.1:8443/health/ready", context=ctx, timeout=5) as response: + if response.status == 200: + print("ready https://127.0.0.1:8443/health/ready") + return + except Exception: + time.sleep(3) + raise SystemExit("public HTTPS endpoint did not become ready") + + +def up(pw): + write_s3_config(pw) + compose("up", "-d", "openbao") + init_openbao(pw) + compose("build", "api-1", "postgres", "rabbitmq") + compose("up", "-d", "--build", "postgres", "minio", "rabbitmq", "valkey") + wait_healthy("postgres", "minio", "rabbitmq", "valkey") + configure_bucket(pw) + configure_rabbit() + compose("up", "-d", "--build", "api-1", "api-2", "worker", "nginx", "otel-edge", "otel-collector", "prometheus", "loki", "tempo", "grafana") + wait_ready() + + +def main(): + ensure_cryptography() + serials = build_material() + pw = material_passwords() + write_configs(pw, serials) + print("local material is in secrets/ and .env; private keys were not printed") + if "--up" in sys.argv: + up(pw) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..3638cd5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Restart user-data stores and the application. OpenBao stays up: restarting it seals the vault. +set -eu +cd "$(dirname "$0")/.." + +wait_healthy() { + i=0 + while [ "$i" -lt 40 ]; do + ok=1 + for svc in "$@"; do + health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}') + if [ "$health" != "healthy" ]; then + ok=0 + fi + done + if [ "$ok" -eq 1 ]; then + return 0 + fi + i=$((i + 1)) + sleep 2 + done + echo "timed out waiting for healthy: $*" >&2 + return 1 +} + +docker compose --env-file .env restart -t 8 postgres minio rabbitmq valkey +wait_healthy postgres minio rabbitmq valkey +docker compose --env-file .env restart -t 8 api-1 api-2 worker +wait_healthy api-1 api-2 worker +docker compose --env-file .env restart -t 3 nginx diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..41877a6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Map the public random alias to the compose service and stop only that replica. +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +found="" +for dir in run/instances/*; do + [ -f "$dir/id" ] || continue + current=$(tr -d '\r\n' < "$dir/id") + if [ "$current" = "$TARGET_INSTANCE_ID" ]; then + found=$(basename "$dir") + break + fi +done +if [ -z "$found" ]; then + echo "unknown instance alias" >&2 + exit 1 +fi +case "$found" in + api-1|api-2) ;; + *) echo "alias does not belong to an API replica" >&2; exit 1 ;; +esac +docker compose --env-file .env stop -t 2 "$found" diff --git a/scripts/track1_run.py b/scripts/track1_run.py new file mode 100644 index 0000000..0e9ce7f --- /dev/null +++ b/scripts/track1_run.py @@ -0,0 +1,312 @@ +"""Track 1: least privilege before and after, then revoke, rotate, and keep a queued job. + +The worker is stopped only for the durability section and started again before exit. +RabbitMQ identity and the worker S3 secret are restored to a working pair. +""" +import json +import secrets +import subprocess +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +S3_CONFIG = ROOT / "secrets" / "seaweed" / "s3.json" +PASSWORDS = ROOT / "secrets" / "operator" / "passwords.json" +EVIDENCE = ROOT / "evidence" +BUCKET = "messenger-attachments" + + +def run(command, check=True, input_text=None): + result = subprocess.run(command, cwd=ROOT, input=input_text, capture_output=True, text=True) + if check and result.returncode != 0: + detail = ((result.stderr or result.stdout or "")[:180]).replace("\n", " ") + raise SystemExit("command failed: " + detail) + return result + + +def psql(sql): + return run( + ["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"], + input_text=sql, + ) + + +def bao_put_worker_s3(access_key, secret_key): + token = json.loads((ROOT / "secrets" / "operator" / "init.json").read_text(encoding="utf-8"))["root_token"] + command = [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "-e", + "BAO_ADDR=https://127.0.0.1:8200", + "-e", + "BAO_CACERT=/openbao/tls/ca.crt", + "-e", + "BAO_CLIENT_CERT=/openbao/tls/operator.crt", + "-e", + "BAO_CLIENT_KEY=/openbao/tls/operator.key", + "-e", + "BAO_TOKEN=" + token, + "openbao", + "bao", + "kv", + "put", + "secret/worker/s3", + "access_key=" + access_key, + "secret_key=" + secret_key, + ] + run(command) + + +def restart_minio(): + run(["docker", "compose", "--env-file", ".env", "restart", "-t", "5", "minio"]) + for _ in range(30): + result = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-minio-1"], check=False) + if (result.stdout or "").strip() == "healthy": + return + time.sleep(2) + raise SystemExit("minio did not become healthy") + + +def track1(): + result = run( + ["docker", "compose", "--env-file", ".env", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "probe", "-m", "app.probes.track1"], + check=False, + ) + line = "" + for item in (result.stdout or "").splitlines(): + if item.startswith("{"): + line = item + if not line: + raise SystemExit("track1 probe produced no report") + return json.loads(line), result.returncode + + +def widen_s3(config): + for identity in config["identities"]: + if identity["name"] == "worker": + identity["actions"] = [ + "Admin", + "Read", + "Write", + "List", + "Read:%s/canary/*" % BUCKET, + "Write:%s/canary/*" % BUCKET, + "List:%s/canary/*" % BUCKET, + ] + return config + + +def worker_secret(config): + for identity in config["identities"]: + if identity["name"] == "worker": + return identity["credentials"][0]["accessKey"], identity["credentials"][0]["secretKey"] + raise SystemExit("worker s3 identity is missing") + + +def set_worker_secret(config, secret_key): + for identity in config["identities"]: + if identity["name"] == "worker": + identity["credentials"][0]["secretKey"] = secret_key + return config + + +def old_secret_denied(access_key, secret_key): + admin = ROOT / "secrets" / "seaweed" / "old-secret.json" + admin.write_text(json.dumps({"access_key": access_key, "secret_key": secret_key}), encoding="utf-8") + code = ( + "import json,boto3\n" + "from botocore.config import Config\n" + "from botocore.exceptions import ClientError\n" + "cred=json.load(open('/c/cred.json'))\n" + "client=boto3.client('s3',endpoint_url='https://minio:8333',aws_access_key_id=cred['access_key']," + "aws_secret_access_key=cred['secret_key'],region_name='us-east-1',verify='/c/ca.crt'," + "config=Config(signature_version='s3v4',s3={'addressing_style':'path'}))\n" + "try:\n" + " client.put_object(Bucket='messenger-attachments',Key='staging/old-secret',Body=b'x')\n" + " print('allow')\n" + "except ClientError as exc:\n" + " print('deny:'+str(exc.response.get('Error',{}).get('Code','error')))\n" + "except Exception as exc:\n" + " print('deny:'+type(exc).__name__)\n" + ) + try: + result = run( + [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7_app", + "-v", + str(ROOT / "secrets" / "pki" / "ca.crt") + ":/c/ca.crt:ro", + "-v", + str(admin) + ":/c/cred.json:ro", + "--entrypoint", + "python", + "mtusi-lab7-messenger:1.7.0", + "-c", + code, + ] + ) + return (result.stdout or "").strip().splitlines()[-1] + finally: + admin.unlink(missing_ok=True) + + +def restore_rabbit(): + run( + ["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "add_user", "worker.messenger.internal", "unused-password"], + check=False, + ) + run( + [ + "docker", + "exec", + "mtusi-lab7-rabbitmq-1", + "rabbitmqctl", + "set_permissions", + "-p", + "/", + "worker.messenger.internal", + "^attachment\\.process\\.v1$", + "^$", + "^attachment\\.process\\.v1$", + ] + ) + + +def queue_depth(): + result = run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "list_queues", "name", "messages"]) + for line in (result.stdout or "").splitlines(): + if line.startswith("attachment.process.v1"): + parts = line.split() + return int(parts[-1]) + return 0 + + +def wait_queue(minimum, timeout=25): + last = 0 + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + last = queue_depth() + if last >= minimum: + return last + time.sleep(1) + return last + + +def upload_while_worker_stopped(): + import sys + + sys.path.insert(0, str(ROOT)) + from tests.client import Client + + client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15) + alice, _bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + return alice, attachment["id"] + + +def wait_status(actor, attachment_id): + import sys + + sys.path.insert(0, str(ROOT)) + from tests.client import Client + + client = Client("https://127.0.0.1:8443", str(ROOT / "secrets" / "pki" / "ca.crt"), timeout=15) + deadline = time.monotonic() + 70 + last = "" + while time.monotonic() < deadline: + current = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"] + last = current["status"] + if last in {"ready", "failed"}: + return last + time.sleep(1) + return last or "timeout" + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + original = S3_CONFIG.read_text(encoding="utf-8") + original_config = json.loads(original) + access_key, old_secret = worker_secret(original_config) + report = {} + committed = False + try: + least, code = track1() + report["least_privilege_exit"] = code + report["least_privilege"] = least + psql("GRANT SELECT ON users TO worker_login;\n") + broad = widen_s3(json.loads(original)) + S3_CONFIG.write_text(json.dumps(broad), encoding="utf-8") + restart_minio() + widened, widened_code = track1() + report["broad_exit"] = widened_code + report["broad"] = widened + S3_CONFIG.write_text(original, encoding="utf-8") + psql("REVOKE SELECT ON users FROM worker_login;\n") + restart_minio() + restored, restored_code = track1() + report["restored_exit"] = restored_code + report["restored"] = restored + + run(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"]) + actor, attachment_id = upload_while_worker_stopped() + uuid.UUID(attachment_id) + report["queued_before_revoke"] = psql( + "SELECT status FROM attachments WHERE id = '%s';\n" % attachment_id + ).stdout.strip() + report["queue_before_revoke"] = wait_queue(1) + run(["docker", "exec", "mtusi-lab7-rabbitmq-1", "rabbitmqctl", "delete_user", "worker.messenger.internal"]) + report["queue_after_revoke"] = queue_depth() + new_secret = secrets.token_hex(24) + rotated = set_worker_secret(json.loads(original), new_secret) + S3_CONFIG.write_text(json.dumps(rotated), encoding="utf-8") + passwords = json.loads(PASSWORDS.read_text(encoding="utf-8")) + passwords["worker_s3_secret"] = new_secret + PASSWORDS.write_text(json.dumps(passwords), encoding="utf-8") + bao_put_worker_s3(access_key, new_secret) + committed = True + restart_minio() + report["old_secret"] = old_secret_denied(access_key, old_secret) + restore_rabbit() + run(["docker", "compose", "--env-file", ".env", "start", "worker"]) + for _ in range(30): + health = run(["docker", "inspect", "--format", "{{.State.Health.Status}}", "mtusi-lab7-worker-1"], check=False) + if (health.stdout or "").strip() == "healthy": + break + time.sleep(2) + report["after_restore"] = wait_status(actor, attachment_id) + report["attachment_id_prefix"] = attachment_id[:8] + finally: + if not committed: + S3_CONFIG.write_text(original, encoding="utf-8") + psql("REVOKE SELECT ON users FROM worker_login;\n") + restore_rabbit() + run(["docker", "compose", "--env-file", ".env", "start", "worker"], check=False) + (EVIDENCE / "track1-run.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps({key: report[key] for key in report if key != "least_privilege" and key != "broad" and key != "restored"})) + print("least", report.get("least_privilege")) + print("broad", report.get("broad")) + print("restored", report.get("restored")) + required = ( + report.get("least_privilege_exit") == 0 + and report.get("broad", {}).get("db_select_password_hash") == "allow" + and str(report.get("broad", {}).get("s3_get_canary", "")).startswith("allow") + and report.get("restored_exit") == 0 + and report.get("queued_before_revoke") == "queued" + and report.get("queue_after_revoke", 0) >= 1 + and str(report.get("old_secret", "")).startswith("deny") + and report.get("after_restore") == "ready" + ) + if not required: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/unseal.py b/scripts/unseal.py new file mode 100644 index 0000000..c6d2029 --- /dev/null +++ b/scripts/unseal.py @@ -0,0 +1,62 @@ +"""Unseal a restarted OpenBao from the operator's local init file. Prints only seal state.""" +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +INIT = ROOT / "secrets" / "operator" / "init.json" + + +def bao(*args): + command = [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "-e", + "BAO_ADDR=https://127.0.0.1:8200", + "-e", + "BAO_CACERT=/openbao/tls/ca.crt", + "-e", + "BAO_CLIENT_CERT=/openbao/tls/operator.crt", + "-e", + "BAO_CLIENT_KEY=/openbao/tls/operator.key", + "openbao", + "bao", + *args, + ] + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True) + + +def status(): + result = bao("status", "-format=json") + text = result.stdout or "" + start = text.find("{") + end = text.rfind("}") + if start < 0 or end < start: + raise SystemExit("OpenBao status is not available") + return json.loads(text[start : end + 1]) + + +def main(): + current = status() + if not current.get("initialized"): + raise SystemExit("OpenBao is not initialized") + if not current.get("sealed"): + print("sealed=false") + return + if not INIT.is_file(): + raise SystemExit("operator init file is missing") + keys = json.loads(INIT.read_text(encoding="utf-8")).get("unseal_keys_b64") or [] + for key in keys[:2]: + result = bao("operator", "unseal", key) + if result.returncode != 0: + raise SystemExit("OpenBao unseal failed") + print("sealed=" + str(bool(status().get("sealed"))).lower()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify_local.py b/scripts/verify_local.py new file mode 100644 index 0000000..1891956 --- /dev/null +++ b/scripts/verify_local.py @@ -0,0 +1,325 @@ +"""Local grade-4 proofs: rehash, session expiry, LB leaf rotation, sealed OpenBao, telemetry. + +Prints statuses only. Restores certificates, the allowlist and the vault before exiting. +""" +import json +import ssl +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + +ROOT = Path(__file__).resolve().parents[1] +PKI = ROOT / "secrets" / "pki" +OPERATOR = ROOT / "secrets" / "operator" +EVIDENCE = ROOT / "evidence" +BASE = "https://127.0.0.1:8443" +CA = PKI / "ca.crt" + + +def run(command, check=True, input_text=None): + result = subprocess.run( + command, + cwd=ROOT, + input=input_text, + capture_output=True, + text=True, + ) + if check and result.returncode != 0: + detail = ((result.stderr or result.stdout or "")[:240]).replace("\n", " ") + raise SystemExit("command failed: " + detail) + return result + + +def bao(*args, token=None): + command = [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "-e", + "BAO_ADDR=https://127.0.0.1:8200", + "-e", + "BAO_CACERT=/openbao/tls/ca.crt", + "-e", + "BAO_CLIENT_CERT=/openbao/tls/operator.crt", + "-e", + "BAO_CLIENT_KEY=/openbao/tls/operator.key", + ] + if token: + command += ["-e", "BAO_TOKEN=" + token] + command += ["openbao", "bao", *args] + return run(command, check=False) + + +def bao_status(): + result = bao("status", "-format=json") + text = result.stdout or "" + start = text.find("{") + end = text.rfind("}") + if start < 0 or end < start: + return None + return json.loads(text[start : end + 1]) + + +def root_token(): + return json.loads((OPERATOR / "init.json").read_text(encoding="utf-8"))["root_token"] + + +def https(path, cert=None, key=None, data=None, method=None, headers=None): + ctx = ssl.create_default_context(cafile=str(CA)) + if cert and key: + ctx.load_cert_chain(str(cert), str(key)) + request = urllib.request.Request(BASE + path, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(request, context=ctx, timeout=8) as response: + return response.status, response.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.read() + + +def psql(sql): + result = run( + ["docker", "exec", "-i", "mtusi-lab7-postgres-1", "psql", "-U", "messenger", "-d", "messenger", "-t", "-A"], + input_text=sql, + ) + return (result.stdout or "").strip() + + +def mtls_ready(cert, key): + code = ( + "import ssl,urllib.request,urllib.error\n" + "ctx=ssl.create_default_context(cafile='/c/ca.crt')\n" + "ctx.load_cert_chain('/c/client.crt','/c/client.key')\n" + "try:\n" + " r=urllib.request.urlopen('https://api-1:8443/health/ready', context=ctx, timeout=8)\n" + " print(r.status)\n" + "except urllib.error.HTTPError as exc:\n" + " print('http', exc.code)\n" + "except Exception as exc:\n" + " print(type(exc).__name__)\n" + ) + result = run( + [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7_app", + "-v", + str(CA) + ":/c/ca.crt:ro", + "-v", + str(cert) + ":/c/client.crt:ro", + "-v", + str(key) + ":/c/client.key:ro", + "--entrypoint", + "python", + "mtusi-lab7-messenger:1.7.0", + "-c", + code, + ] + ) + return (result.stdout or "").strip() + + +def issue_lb_leaf(): + ca_key = serialization.load_pem_private_key((PKI / "ca.key").read_bytes(), password=None) + ca_cert = x509.load_pem_x509_certificate((PKI / "ca.crt").read_bytes()) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now = __import__("datetime").datetime.now(__import__("datetime").timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "lb.messenger.internal")])) + .issuer_name(ca_cert.subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - __import__("datetime").timedelta(minutes=5)) + .not_valid_after(now + __import__("datetime").timedelta(days=7)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("lb.messenger.internal")]), critical=False) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + leaf_key = PKI / "lb-rotated.key" + leaf_crt = PKI / "lb-rotated.crt" + leaf_key.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + leaf_crt.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + return leaf_crt, leaf_key, format(cert.serial_number, "X") + + +def reload_nginx(): + run(["docker", "exec", "mtusi-lab7-nginx-1", "nginx", "-s", "reload"]) + + +def count_leaks(): + patterns = ("Bearer ", "password=", "postgresql://", "aws_secret_access_key", "BEGIN PRIVATE") + counts = {pattern: 0 for pattern in patterns} + for name in ("mtusi-lab7-api-1-1", "mtusi-lab7-api-2-1", "mtusi-lab7-worker-1", "mtusi-lab7-loki-1"): + result = run(["docker", "logs", name, "--since", "20m"], check=False) + text = (result.stdout or "") + (result.stderr or "") + for pattern in patterns: + counts[pattern] += text.count(pattern) + return counts + + +def rehash_and_expiry(): + from argon2 import PasswordHasher + + username = "rehash" + uuid.uuid4().hex[:8] + password = "rotate-horse-2" + body = json.dumps({"username": username, "display_name": "Rehash", "password": password}).encode("utf-8") + status, _ = https("/api/v1/users", data=body, method="POST", headers={"Content-Type": "application/json"}) + if status != 201: + raise SystemExit("rehash register failed") + old = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1).hash(password) + psql("UPDATE users SET password_hash = %s WHERE username = %s;\n" % (_quote(old), _quote(username))) + before = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username)) + import base64 + + basic = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + login_status, raw = https("/api/v1/auth/sessions", method="POST", headers={"Authorization": "Basic " + basic}) + after = psql("SELECT substring(password_hash from 'm=[0-9]+,t=[0-9]+,p=[0-9]+') FROM users WHERE username = %s;\n" % _quote(username)) + token = json.loads(raw.decode("utf-8")).get("token") if login_status == 201 else "" + expired = None + if token: + psql( + "UPDATE sessions SET expires_at = NOW() - INTERVAL '1 minute' " + "WHERE user_id = (SELECT id FROM users WHERE username = %s);\n" % _quote(username) + ) + expired, _ = https("/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + psql("DELETE FROM sessions WHERE user_id IN (SELECT id FROM users WHERE username = %s);\n" % _quote(username)) + psql("DELETE FROM users WHERE username = %s;\n" % _quote(username)) + return { + "login": login_status, + "params_before": before, + "params_after": after, + "expired_session": expired, + } + + +def _quote(value): + return "'" + value.replace("'", "''") + "'" + + +def _until(action, expected, attempts=8): + last = "" + for _ in range(attempts): + last = action() + if last == expected or last.startswith(expected): + return last + time.sleep(1) + return last + + +def rotate_leaf(): + allow = PKI / "lb-allowed-serials.txt" + backup_allow = allow.read_bytes() + backup_crt = (PKI / "lb.crt").read_bytes() + backup_key = (PKI / "lb.key").read_bytes() + leaf_crt, leaf_key, serial = issue_lb_leaf() + try: + allow.write_text(serial + "\n", encoding="ascii") + rotated = _until(lambda: mtls_ready(leaf_crt, leaf_key), "200") + retired = _until(lambda: mtls_ready(PKI / "lb.crt", PKI / "lb.key"), "http 403") + (PKI / "lb.crt").write_bytes(leaf_crt.read_bytes()) + (PKI / "lb.key").write_bytes(leaf_key.read_bytes()) + reload_nginx() + time.sleep(1) + public, _ = https("/health/ready") + return {"new_serial_ready": rotated, "old_serial": retired, "public_after_reload": public} + finally: + allow.write_bytes(backup_allow) + (PKI / "lb.crt").write_bytes(backup_crt) + (PKI / "lb.key").write_bytes(backup_key) + leaf_crt.unlink(missing_ok=True) + leaf_key.unlink(missing_ok=True) + reload_nginx() + + +def seal_cycle(): + before, _ = https("/health/ready") + open_login, _ = https( + "/api/v1/auth/sessions", + method="POST", + headers={"Authorization": "Basic YmFkOmJhZA=="}, + ) + sealed = bao("operator", "seal", token=root_token()) + if sealed.returncode != 0: + raise SystemExit("OpenBao seal failed") + report = {"ready_before": before, "login_before_seal": open_login, "login_while_sealed": None, "secret_env": []} + try: + time.sleep(20) + deadline = time.monotonic() + 30 + denied = None + while time.monotonic() < deadline: + denied, _ = https( + "/api/v1/auth/sessions", + method="POST", + headers={"Authorization": "Basic YmFkOmJhZA=="}, + ) + if denied == 503: + break + time.sleep(2) + env = run(["docker", "exec", "mtusi-lab7-api-1-1", "printenv"], check=False) + leaked = [ + line.split("=", 1)[0] + for line in (env.stdout or "").splitlines() + if line.startswith(("DATABASE_URL=", "S3_SECRET_ACCESS_KEY=", "VALKEY_PASSWORD=", "BAO_TOKEN=")) + ] + report["login_while_sealed"] = denied + report["secret_env"] = leaked + finally: + run(["python", "scripts/unseal.py"]) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + status, _ = https("/health/ready") + if status == 200: + break + time.sleep(2) + return report + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + report = { + "rehash": rehash_and_expiry(), + "leaf": rotate_leaf(), + "telemetry_leak_counts": count_leaks(), + } + sealed = seal_cycle() + report["sealed"] = sealed if isinstance(sealed, dict) else {"restored": True} + ready, _ = https("/health/ready") + report["ready_after_unseal"] = ready + (EVIDENCE / "verify-local.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report)) + ok = ( + report["rehash"]["login"] == 201 + and report["rehash"]["params_before"].startswith("m=8192") + and report["rehash"]["params_after"] == "m=19456,t=2,p=1" + and report["rehash"]["expired_session"] == 401 + and report["leaf"]["new_serial_ready"] == "200" + and report["leaf"]["old_serial"].startswith("http 403") + and report["leaf"]["public_after_reload"] == 200 + and report["sealed"]["login_before_seal"] == 401 + and report["sealed"]["login_while_sealed"] == 503 + and report["sealed"]["secret_env"] == [] + and ready == 200 + and all(value == 0 for value in report["telemetry_leak_counts"].values()) + ) + if not ok: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..b63c49e 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,41 @@ -# Модель угроз — заполнить +# Модель угроз + +Учебный стенд с синтетическими пользователями. Атакующий — процесс в сети `app` или внешний клиент `https://127.0.0.1:8443`. Он не имеет root token OpenBao, unseal keys и ключа CA: они лежат только в `secrets/operator` и `secrets/pki/ca.key` на машине оператора и не монтируются в контейнеры приложения. ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +| Актив | Где живёт | Кто легитимно читает | +| --- | --- | --- | +| Пароль и Argon2id-хеш | PostgreSQL, колонка `users.password_hash` | Только `api_login` при входе. В ответе API и в telemetry хеша нет | +| Сессионный токен | Клиент хранит сырое значение; сервер — SHA-256 в `sessions.token_hash` | Предъявитель Bearer. `X-User-Id` не меняет сессию | +| Сообщения и вложения | PostgreSQL и приватный bucket `messenger-attachments` | Участники чата через API. Объект S3 наружу не подписывается | +| Service identity | Leaf-сертификат роли, монтируется в контейнер | Процесс этой роли. Чужой CN того же CA не равен этой роли | +| Root token и unseal keys | `secrets/operator/init.json` у оператора | Только оператор. Приложение его не видит | +| Telemetry | Collector → Prometheus / Loki / Tempo | Операторы Grafana. В пайплайне вырезаются Authorization, URL и SQL | + +Акторы: участник чата, посторонний зарегистрированный пользователь, внешний клиент без сертификата, скомпрометированный worker, оператор стенда. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +Публичная граница — TLS ingress (`nginx`, `127.0.0.1:8443`). Внутренняя граница API — mTLS на `api-*:8443`: сертификат клиента обязателен, SAN должен быть `lb.messenger.internal`, серийный номер должен быть в `lb-allowed-serials.txt`. Порт приложения `8080` слушает только `127.0.0.1` внутри контейнера. Связь с зависимостями описана в [contracts/trust-matrix.md](../contracts/trust-matrix.md). + +OpenBao не в dev-mode. Процесс API/worker логинится своим leaf через `auth/cert`, получает token с policy `api` или `worker` и TTL 20 минут, продлевает его сам. Корневого токена в окружении контейнера нет. | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | +| Внешний клиент подставляет свой или пустой CA и читает `/health/ready` или API | Клиент доверяет только стендовому CA и имени `localhost` | Сертификат ingress выписан локальным CA, SAN `localhost` и `127.0.0.1`; клиентские скрипты не отключают проверку | `make test-security` и probe `ingress` | Учебный CA лежит у оператора; компрометация `ca.key` позволяет выпустить любой leaf | +| Клиент без сертификата, с чужим CA или с сертификатом worker/stranger открывает TCP к `api-1:8443` | В API входит только LB | TLS требует client cert; код прокси сверяет SAN и serial allowlist. Порт 8080 не маршрутизируется | `python -m app.probes.security`: no cert, foreign CA, worker SAN, stranger SAN, bypass `:8080` | Уже установленное соединение живёт до закрытия; новый handshake видит новый allowlist | +| Участник Eve читает чат Alice или подставляет `X-User-Id` | Чужая история и вложения неразличимы от отсутствующих, сессия не подменяется заголовком | Сначала Bearer, затем membership; чужой объект — 404; `X-User-Id` игнорируется | Публичный `make test` и сценарий logout/expiry | Список участников после создания чата не меняется: такого API нет | +| Подбор пароля или дорогой login DoS на обе реплики | Argon2id не должен быть бесплатным усилителем | Общий счётчик Valkey до KDF: 10/мин на имя и 60/мин на стенд, ответ 429 и `Retry-After`. Пароль в ошибке и логе не появляется | Probe `login-rate-limit`; измерение `evidence/argon2-budget.json` | Лимит по имени, а не по IP: за LB все клиенты выглядят одинаково | +| Worker читает `password_hash`, пишет в `canary/` или публикует в очередь | Компрометация worker не равна компрометации API | Отдельные DB role, S3 policy и RabbitMQ CN. Worker не получает secret API | `python scripts/track1_run.py`: узкие права, временное расширение, возврат, отзыв CN и смена S3 secret при queued-задаче | Worker всё ещё читает байты вложений, которые ему поручили обработать | +| Процесс предъявляет root token или читает чужой KV | Секреты выдаются по роли и с TTL | Cert auth, policy без `sys/` и без записи, отказ от policy `root`. Пока OpenBao sealed, login отвечает 503; `/health/ready` проверяет тот же `healthy()`. В окружении контейнера нет `DATABASE_URL`, ключей S3 и `BAO_TOKEN` | `python scripts/verify_local.py` | Уже загруженные в память DSN живут до рестарта процесса | -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +## Lifecycle -## Lifecycle и инциденты +Оператор выпускает CA и leaf командой `python scripts/bootstrap.py`. Ключ CA и `init.json` остаются на хосте. После рестарта контейнера OpenBao оператор выполняет `python scripts/unseal.py`: API при этом перезапускать не нужно, пока vault не запечатан. Unseal shares в контейнер OpenBao не монтируются. -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +Ротация leaf LB выполнена без пересборки образа: новый сертификат с тем же SAN `lb.messenger.internal`, в allowlist оставлен только новый serial. Прямой запрос к `api-1:8443` дал 200 новому serial и HTTP 403 старому. Nginx перечитал сертификат через `nginx -s reload` и публичный `/health/ready` остался 200. После замера исходные `lb.crt`, `lb.key` и allowlist возвращены. + +Ротация секрета S3 worker: новое значение записано в `s3.json` SeaweedFS и в KV `secret/worker/s3`, контейнер `minio` перезапущен, образ не пересобирался. Старый secret получил `SignatureDoesNotMatch`. Пользователь RabbitMQ worker на время удалён, сообщение очереди сохранилось, после возврата identity вложение из `queued` стало `ready`. + +Истечение client-сессии даёт 401 и `WWW-Authenticate: Bearer`. Истечение token OpenBao приводит к повторному cert login; если leaf уже удалён из роли, ready остаётся 503. Остановка OpenBao не переключает приложение на пароль из переменной окружения: такой переменной у API нет. diff --git a/security/scan-triage.md b/security/scan-triage.md new file mode 100644 index 0000000..b65a452 --- /dev/null +++ b/security/scan-triage.md @@ -0,0 +1,17 @@ +# Разбор зависимостей + +Дата прогона: 2026-11-07. Команда: `python -m pip_audit -r requirements.txt`. В трёх пакетах 84 записи: Starlette 0.45.3 — 14, Pillow 11.1.0 — 69, opentelemetry-api 1.29.0 — 1. У uvicorn, psycopg, argon2-cffi, boto3, pika и redis записей нет. + +Образ приложения — `mtusi-lab7-messenger:1.7.0`, в него копируется тот же `requirements.txt`. `docker scout` 1.20.1 установлен, но без входа в Docker Hub базу CVE не открывает. `trivy` в PATH нет, поэтому слой Debian отдельно не разбирался. + +## Что вызывает процесс + +Starlette указан в зависимостях и в коде не импортируется. HTTP-сервер — свой ASGI в `app/asgi.py` под uvicorn. Разбор multipart, `FileResponse`, `StaticFiles` и `HTTPEndpoint` на запрос не выходят, поэтому записи Starlette про форму, Range, Host и Windows-пути к текущему API не применяются. + +`opentelemetry-api` CVE-2026-114603 относится к разбору входящего заголовка `tracestate`. Публичный ASGI этот заголовок в контекст не кладёт. В очередь попадает carrier, который API собрал сам через `inject`. + +Pillow вызывается после проверки сигнатуры. `detect_image` принимает только PNG и JPEG, тело ограничено 10 МиБ, `Image.MAX_IMAGE_PIXELS` равен 40 000 000, `DecompressionBombError` превращается в 413. Дальше `exif_transpose`, resize и сохранение JPEG. На этом пути имеют смысл дефекты декодера PNG/JPEG и разбора EXIF, в том числе CVE-2026-112403 (буфер JPEG) и CVE-2026-96643 (квадратичный EXIF). Записи про PSD, FITS, PDF, шрифты, TIFF, WMF и Ghostscript этот путь не открывает: такие файлы отсекаются по сигнатуре до `Image.open`. + +## Что оставлено + +Линейки Starlette и Pillow не поднимались. Исправления уводят Starlette на 0.47+ и Pillow на 12/13, а вызываемый код Starlette в процессе отсутствует. Для Pillow остаются уже стоящие ограничители размера, пикселей и CPU/RAM worker. Повторный `pip-audit` нужен, если приложение начнёт импортировать Starlette или принимать другие форматы изображений.