WIP: ЛР 7 — БВТ2301 — Маслов — на 5 #26

Closed
wh1tet3a wants to merge 2 commits from wh1tet3a/mtusi2026lab7:solution/bvt2301-maslov into main
155 changed files with 17470 additions and 82 deletions
+7
View File
@@ -18,3 +18,10 @@ __pycache__
node_modules
evidence
.DS_Store
operator/
trust/
.work*
.pytest_cache/
.ruff_cache/
student-unit/
+19 -14
View File
@@ -1,18 +1,23 @@
## Лабораторная 7
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Маслов, БВТ2301.
Целевая оценка: 5, трек 2 — жизненный цикл сертификатов и секретов.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
**Черновик. Работа пока не готова к сдаче.**
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Перенесено собственное приложение ЛР6: API, очередь задач, обработка изображений отдельным worker, наблюдаемость. Добавлены TLS/mTLS, раздельные удостоверения и права сервисов, конфигурация OpenBao, короткие сертификаты и токены, операторские команды ротации и восстановления, ограничения входа перед хешированием пароля, обновление старого хеша при успешном входе. Границы доверия и ограничения описаны в REPORT.md, contracts/trust-matrix.md и security/THREAT_MODEL.md.
Пройдены 29 unit-тестов, проверки структуры/OpenAPI/собственного Python-кода и конфигурации Compose. Первый вариант образов собран. pip-audit проверил 40 закреплённых runtime-зависимостей Python без известных уязвимостей по своей базе на момент запуска. Это не проверка безопасности образов.
Не завершены: запуск всего защищённого стенда, HTTP-регрессия, положительные и отрицательные проверки mTLS/прав, исследование ротации/отзыва/истечения сертификатов и замены CA под нагрузкой, проверка образов и воспроизведение из чистого клона. Пока нет результатов, подтверждающих выполнение требований на оценку 5.
Запуск оператора: `uv sync --locked`, `make up PYTHON=.venv/bin/python`. Предполагаемый API: `https://localhost:8443`; доверенный CA создаётся локально в `trust/ca.pem`. Команды и подробности: [REPORT.md](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab7/src/branch/solution/bvt2301-maslov/REPORT.md).
Источники: задание, собственная ЛР6, официальная документация OpenBao/HAProxy/OWASP. Использована помощь ИИ при проектировании, реализации, проверках и подготовке отчёта; чужие студенческие решения не использовались.
- [ ] Весь обязательный минимум подтверждён запуском и регрессией.
- [x] Контракт и публичные тесты сохранены.
- [ ] make test и сценарии безопасности выполнены на защищённом стенде.
- [ ] Исследование выбранного трека выполнено и измерения приложены.
- [x] Секреты и приватные ключи находятся вне Git.
- [x] README задания сохранён, собственный REPORT.md заполнен.
+6
View File
@@ -30,3 +30,9 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
operator/
trust/
.work*
.pytest_cache/
.ruff_cache/
+28
View File
@@ -0,0 +1,28 @@
FROM library/golang:1.27.4-alpine@sha256:70bd7ddea8b6cbe5d6943cde279d6b67feb10cc924fb9100d1d8757c1575b280 AS gateway
WORKDIR /source
COPY infra/tls/*.go ./
RUN CGO_ENABLED=0 GOMAXPROCS=2 go build -trimpath -ldflags="-s -w" -o /out/tls-gateway gateway.go
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=uv /uv /usr/local/bin/uv
WORKDIR /app
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ARG VCS_REF=unknown
LABEL org.opencontainers.image.revision=$VCS_REF
WORKDIR /app
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
COPY --from=builder /app/.venv /app/.venv
COPY --from=gateway /out/tls-gateway /usr/local/bin/tls-gateway
COPY src /app/src
COPY migrations /app/migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+17
View File
@@ -31,3 +31,20 @@ test-failover:
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
UV ?= uv
CA_FILE = trust/ca.pem
.PHONY: bootstrap up down lint unit test-student
bootstrap:
$(UV) run --locked python -m scripts.bootstrap_security
up:
$(UV) run --locked python -m scripts.up_security --build
down:
docker compose down
lint:
$(UV) run --locked ruff check src student-tests student-unit scripts --exclude scripts/check_contract.py --exclude scripts/validate_openapi.py
$(UV) run --locked ruff format --check src student-tests student-unit scripts --exclude scripts/check_contract.py --exclude scripts/validate_openapi.py
unit:
$(UV) run --locked python -m pytest student-unit -q
test-student:
BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" $(UV) run --locked python -m pytest student-tests
+27 -24
View File
@@ -1,37 +1,40 @@
# Отчёт — лабораторная 7
# Лабораторная 7 — Маслов, БВТ2301
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Цель — оценка 5, трек 2: жизненный цикл PKI и секретов. Собственный код ЛР6 перенесён поверх независимого skeleton ЛР7. README/COURSE/публичные tests/OpenAPI/check/validate сохранены; trust matrix заполнена как требует задание.
## Запуск из чистого clone
Текущий статус: реализация готовится к первому запуску. Пройдены 29 unit tests, offline/OpenAPI/lint, проверка конфигурации Compose. Сканирование pip-audit2.10.1 проверило 40 закреплённых runtime зависимостей и не нашло известных уязвимостей в своей базе на момент запуска (`evidence/dependency-scan-first.json`). Это не результат проверки образов или полного стенда. HTTP/mTLS/OpenBao/ротация/исследование/чистый clone пока не подтверждены.
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
## Запуск оператора
## Архитектура и решения
Нужны Docker Desktop/Compose v2, Git и uv0.13.7. `uv sync --locked`, затем `make up PYTHON=.venv/bin/python`. Команда создаёт локальные CA/secrets, собирает образы и явно выполняет init/unseal/configure. Повторный `make up` сохраняет существующие ключи, пароли и состояние. Для обычного рестарта API операторский init/unseal не нужен: отдельные role agents продолжают renewal.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
API `https://localhost:8443`, Grafana `https://localhost:3447`, S3 operator endpoint `https://localhost:9007`, OpenBao `https://localhost:8207`. Все host binds только 127.0.0.1. Grafana admin password — локальный `secrets/grafana_password`; операторские файлы находятся в `operator/`, не в Git. Проверка API: `make test test-security CA_FILE=trust/ca.pem PYTHON=.venv/bin/python`; полный собственный HTTP suite: `make test-student`; матрица: `uv run --locked python -m scripts.verify_security`.
## Доказательства критериев
Остановка `docker compose down` сохраняет named volumes. Не удалять operator/ и volumes по отдельности: CA, unseal shares и сохранённый Bao должны соответствовать друг другу. `python -m scripts.bao_operator status/unseal` использует проверяемый TLS и operator client cert; токены/shares не печатаются. При длительном seal/expiry требуется явный operator recovery, а не отключение TLS.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
## Архитектура и границы
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
TLS1.3, CA/SAN, mTLS с exact URI, раздельные роли БД/S3/Valkey и scoped cert auth/KV/PKI в OpenBao. Полное покрытие, порты, TTL, локальные transport boundaries и оправдание исключений: [trust matrix](contracts/trust-matrix.md). Угрозы и отрицательные проверки: [THREAT_MODEL](security/THREAT_MODEL.md). Конфиги gateway — `infra/tls`; политики создаёт `scripts/bao_operator.py`.
## Эксперименты и отказы
Root CA private key остаётся у оператора; intermediate key создаётся внутри persistent non-dev OpenBao. Auth tokens короткие, каждое приложение получает только собственные secrets, named volumes разделены по ролям. Snapshot secret имеет конечный срок и проверяется на API request/получении DB connection. S3 credentials читаются при каждой операции; DB conninfo — при новом подключении. Это статические рабочие PG/S3 accounts с управляемой ротацией, не заявляется динамический DB plugin. Сетевой доступ дополнительно требует короткого клиентского cert.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Пароли Argon2id: t3/m65536KiB/p4, 2 KDF slots/API. При успешном входе старый hash обновляется через compare-and-swap; неверный пароль не запускает rehash. SQL limiter общий для реплик: global burst8/refill2/s и account8/min, 429/Retry-After перед KDF. Хеши/session digest/credentials не экспортируются. JSON ограничен 64KiB/5с приёма; изображения и upload-worker сохраняют пределы ЛР5–6.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Наблюдаемость и устойчивый pipeline сохранены: API принимает файл и jobs/outbox устойчиво, отдельный relay/worker, версии и fencing, конечные retries, ручной replay и cleanup. Collector сохраняет bounded queues и limiter; Grafana provisioning, три persistent хранилища и SLO rules перенесены. Полные повторные проверки ещё предстоят; результаты ЛР6 не выдаются за результаты ЛР7.
## Ограничения и следующие шаги
## Исследование на 5
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Гипотеза: автоматическое обновление leaf и service tokens позволяет использовать короткие TTL; dual trust даёт безопасно заменить CA, а expiry/revocation приводят к ограниченному отказу без plaintext fallback. План: фиксированная синтетическая нагрузка, renewal, ротация S3 secret и leaf без image rebuild, отзыв старого leaf с новым TLS подключением, expiry/seal и восстановление, CA rollover с retirement старого trust. Измеряются ошибки, время восстановления, сохранность всех подтверждённых сообщений/заданий и отсутствие утечек. Результаты и ограничения заполняются после реального опыта.
## Ответы на контрольные вопросы
1. TLS шифрует поток и проверяет сервер через CA и SAN. В mTLS сервер также проверяет клиентский сертификат. Gateway затем сопоставляет URI с разрешённой ролью, а SQL/S3/broker/Bao ACL ограничивает конкретное действие. Проверка сертификата и разрешение операции — отдельные шаги.
2. CA может выдать удостоверение worker, API и LB. Подпись подтверждает доверенного издателя, но worker не становится LB: API принимает только URI lb. Поэтому нужен отдельный тест с чужой ролью, подписанной правильным CA.
3. Первое удостоверение создаёт локальный оператор из offline CA. Только собственный agent получает bootstrap leaf; приложения не получают CA private key/root token/unseal. Unseal выполняет оператор двумя shares из трёх защищённых файлов. В учебном стенде это один оператор, без имитации независимых хранителей.
4. Пока agent имеет действующий cert/token и Bao доступен, он обновляет удостоверение и секреты. При недоступном/sealed Bao обновление прекращается; cache имеет конечный срок и затем API/новые подключения отказывают. Обычный рестарт API использует продолжающий работать agent. После длительного истечения всех удостоверений оператор восстанавливает trust через явно описанный recovery.
## Источники и ограничения
Задание и собственная ЛР6; [OpenBao cert auth](https://openbao.org/docs/auth/cert/), [TLS listener](https://openbao.org/docs/configuration/listener/tcp/), [политики](https://openbao.org/docs/concepts/policies/), [OWASP password storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [HAProxy3.2](https://docs.haproxy.org/3.2/configuration.html). Точные versions/digests находятся в Compose/lock. При проектировании, реализации, проверках и подготовке отчёта использована помощь ИИ; чужие студенческие решения/истории/измерения не использовались.
Один локальный host и одиночные PG/S3/broker/OpenBao — ограничения доступности. Нет защиты от host root/потери диска. Короткие TTL демонстрационные. Финальные runtime результаты, image scan/triage и воспроизводимость не подтверждаются одним наличием конфигов.
+36
View File
@@ -0,0 +1,36 @@
services:
api:
environment:
PIPELINE_FAULTS: '1'
relay:
environment:
PIPELINE_FAULTS: '1'
worker:
environment:
PIPELINE_FAULTS: '1'
controller:
image: ${API_IMAGE:-mtusi-messenger:lab7}
pull_policy: never
entrypoint:
- python
user: 10001:10001
environment:
IDENTITY_ENABLED: '1'
volumes:
- ./secrets/operator-task:/run/identity:ro
- ./trust:/trust:ro
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /tmp:rw,noexec,nosuid,size=8m
cpus: 0.5
mem_limit: 192m
pids_limit: 64
healthcheck:
disable: true
command:
- -c
- import time;time.sleep(86400)
+1563
View File
File diff suppressed because it is too large Load Diff
+33 -28
View File
@@ -1,37 +1,42 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия — Маслов, БВТ2301
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
Один локальный стенд, отдельная сеть Compose. В сетевых соединениях используется TLS 1.3 с проверкой цепочки и DNS/IP SAN. Удостоверение сервиса — URI `spiffe://messenger/<роль>`. Gateway проверяет текущий CA, срок сертификата, список отозванных серийных номеров и разрешённую роль; HTTP дополнительно ограничен методом и путём. Конфигурации находятся в `infra/tls/*.json`. Наличие конфигурации не заменяет проверку: фактические результаты должны быть в `evidence/security-*.json` и REPORT.
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Связь | Endpoint и защита | Разрешение и срок | Проверка |
|---|---|---|---|
| Клиент → LB | `https://localhost:8443`, CA `trust/ca.pem`, SAN localhost/127.0.0.1 | Bearer, сессия 3600 с; общий SQL limiter до Argon2 | `make test test-security CA_FILE=trust/ca.pem`; HTTP suite |
| LB → все API | `api:8443`, mTLS; DNS SAN api; только URI lb | Роль lb, без обхода на API:8080; leaf 120 с | `python -m scripts.verify_security`: все адреса DNS API, правильный / отсутствующий / чужой CA / другая доверенная роль |
| API/worker/relay/observer/reconciler → PostgreSQL | `postgres:5443`, PG SSLRequest + mTLS, `sslmode=verify-full`, `gssencmode=disable` | Разные `svc_<роль>`, SCRAM, без SUPERUSER/CREATEROLE; нет доступа worker к users/sessions; TLS соединение ≤45 с | operator probe: SELECT 1 и запрещённые SQL; `scripts/prepare_dependencies.py` |
| API/worker/reconciler → S3 | `https://s3:9443`, mTLS + отдельные SigV4 accounts | API: PUT staging/GET objects; worker: GET objects/PUT versioned; reconciler: LIST/GET/DELETE objects; нет admin | operator probe: разрешённые PUT/GET, запрещённые prefix/DELETE/admin; собственные тестовые объекты удаляются |
| API → broker | N/A: API устойчиво пишет jobs/outbox в PG; producer — отдельный relay | API не получает broker password и не входит в allowlist broker | API-cert → broker должен быть отклонён |
| Relay/worker → broker | `broker:6380`, mTLS; разные URI relay/worker + разные Valkey ACL | relay XADD/WAITAOF; worker XGROUP/XREADGROUP/XAUTOCLAIM/XACK/XDEL; нет CONFIG; соединение ≤45 с | operator probe: PING после auth, CONFIG denied, три отрицательных TLS случая |
| API/worker/relay/observer → Collector | `otel:4438`, mTLS; только перечисленные URI | Только POST `/v1/`; отдельные идентичности API и worker | operator probe: реальный OTLP POST и неверные cert; pipeline в `infra/otel/collector.yaml` |
| Collector → Loki/Tempo | `loki:3310/otlp`, `tempo:4438/v1/traces`, mTLS, URI otel | Только ingest; Grafana не может писать ingest | Gateway allowlist и проверки матрицы; bounded exporter queues сохранены |
| Grafana → Prometheus/Loki/Tempo | `prometheus:9900`, `loki:3310`, `tempo:3320`, mTLS, URI grafana | Запросы; нет ingest/операторских изменений | Реальные datasource queries и запрещённые методы |
| Prometheus → Collector/LB/Loki/Tempo | `otel:18889`, `otel:18888`, `lb:18404`, `loki:3310`, `tempo:3320`, mTLS | Только GET metrics; URI prometheus | Prometheus up/actual series и negative route tests |
| Role agents → OpenBao | `https://openbao:8200`, native mTLS + cert auth с exact CN и URI | Только own KV v2 path и own PKI role. Auth token 120 с, max 600 с; bounded KV snapshot ≤60 с; рабочий leaf 120 с | operator probe: own secret 200, чужой secret/policy write/foreign PKI 403; renewal и expiry research |
| Оператор → OpenBao/Grafana/S3 | localhost `8207`/`3447`/`9007`, проверяемый TLS; Bao/S3 также operator client cert | Scoped operator policy; Grafana Basic; S3 отдельный root account только у оператора | CLI оператора; app/worker не монтируют operator/ и admin credentials |
| API → cache | `cache:6380`, mTLS URI api + Valkey ACL | Только GET/SET `user:*`, TTL карточки 30 с; нет sessions/passwords в cache | TLS/ACL probe, публичные бизнес-тесты |
| Репликация БД/S3/broker | N/A: одиночные PG/S3/broker с persistent volumes | Отказ реплики API переносится; отказ хоста/диска остаётся ограничением | HTTPS failover/persistence; не заявляется HA хранилищ |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
## Закрытая локальная граница
## Обязательные отрицательные проверки
Native listener каждого сервиса доступен только на 127.0.0.1 в его network namespace. Gateway разделяет namespace через `network_mode: service:<имя>`; API запускает gateway и Python сервер в одном контейнере. Прямые 8080/5432/6379/9000/4318/8889/9090/3100/3200/3000/8404 проверяются из отдельного контейнера и должны отказывать. На host опубликованы только 8443 и локальные административные TLS endpoints.
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
Часть продуктов умеет native TLS, но единый gateway выбран для одинакового обновления коротких сертификатов и точной проверки URI/отзыва. Локальные HTTP выходы Collector/Grafana/Prometheus и OTLP SDK тоже слушают только 127.0.0.1: gateway на каждом новом сетевом соединении читает текущие сертификат и CA, проверяет SAN и server URI. Это явно ограниченный локальный HTTP hop; сетевые границы шифруются. Публичного HTTP fallback нет. PostgreSQL native Unix socket остаётся только для operator bootstrap внутри контейнера PG.
## Bootstrap и эксплуатация
## Bootstrap, renewal и отзыв
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
`make up` выполняет явные шаги оператора. Root CA private key находится в `operator/` (0700, файлы 0600) и не монтируется в контейнеры. Root CA — 365 дней; intermediate — 30 дней, его ключ создаётся внутри OpenBao. Bootstrap cert сервиса — 30 минут, доступен только собственному agent; рабочий ключ сохраняется в собственном named volume (0700/0600). Обычный рестарт API использует продолжавший работать отдельный agent.
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
OpenBao работает с persistent file storage, вне dev-mode. Init создаёт 3 shares, threshold 2; файлы остаются у оператора, не монтируются в Bao/apps. В учебном стенде один оператор хранит все три защищённых файла; разделение между независимыми держателями не моделируется. Первичный root token используется только для установки auth/policies/PKI/KV/audit, затем отзывается и удаляется. У оператора cert-auth с отдельной policy и коротким token.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
Каждая роль получает только собственные KV и PKI разрешения. Agent опрашивает Bao каждые 15 с, обновляет leaf до истечения, сохраняет пары через атомарную смену каталога; snapshot secrets имеет конечный срок. После expiry запросы/новые подключения завершаются контролируемым отказом. Длительная остановка/seal Bao требует явного operator recovery; статического бессрочного секрета для автоматического обхода нет.
Сертификат сервера Bao тоже короткий и обновляется native `tls_auto_reload`; CA bundle при rollover перечитывается через явный restart/unseal. HAProxy перечитывает leaf через master-worker SIGUSR2, без пересборки. Отзыв проверяется новым TLS подключением; HTTP закрывает соединение после ответа, PG/broker ограничены 45 с и сроком клиентского cert. Список serial deny не подменяет отзыв token/рабочего пароля при компрометации всей роли: эти действия оператор выполняет отдельно.
## Права файлов и исключения
Локальная `secrets/` имеет 0700. Отдельные bind files читаемы сервисным UID Docker Desktop; каждому agent смонтирован только его bootstrap каталог. Рабочие keys/snapshots в named volumes имеют 0600 и читаются собственным UID. App/worker — UID10001, read_only, drop ALL, no-new-privileges, bounded tmpfs/pids/CPU/RAM. Stateful данные находятся в отдельных томах.
Одноразовый identity-init использует UID0 и только CHOWN/DAC_OVERRIDE для начальной подготовки томов; после завершения не работает. Официальный PG entrypoint использует ограниченный набор прав для init/drop-uid. Operator probe запускается только явно с profile operator, read_only и DAC_READ_SEARCH для чтения проверяемых service identities; это инструмент оператора, в приложение он не встраивается и root CA/unseal туда не передаются. Docker socket, host network и privileged отсутствуют.
+9
View File
@@ -0,0 +1,9 @@
{
"scanner": "pip-audit 2.10.1",
"scope": "locked runtime dependencies, no dev group",
"lock_sha256": "0fb0fa939296df200ddeda2ad6379a9f1e99ff08fbcfc0ec8020b2a1115cf820",
"scanned_dependencies": 40,
"vulnerable_dependencies": [],
"scan_completed": true,
"recorded_at": "2026-11-20T15:08:38.146220+00:00"
}
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+789
View File
@@ -0,0 +1,789 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер: диагностика и SLO",
"schemaVersion": 41,
"version": 1,
"editable": false,
"timezone": "utc",
"refresh": "5s",
"time": {
"from": "now-30m",
"to": "now"
},
"tags": [
"lab6",
"slo"
],
"panels": [
{
"id": 1,
"title": "Запросы через балансировщик, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m]))",
"refId": "A",
"legendFormat": "все бизнес-запросы",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Health requests use a separate backend and are excluded. Includes ingress 503 with no API available.",
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "Доля ответов 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[1m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m])),0.001)",
"refId": "A",
"legendFormat": "ошибки 5xx",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "API: p50 / p95 / p99, секунды",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Quantiles from combined histogram buckets; complete ASGI response, no average of replica quantiles.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"title": "Очередь: ожидающие задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_queue_depth)",
"refId": "A",
"legendFormat": "ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Logical PostgreSQL jobs in queued state, not count of duplicate stream notifications.",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "Возраст старейшего незавершённого задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max(messenger_oldest_job_age_seconds)",
"refId": "A",
"legendFormat": "возраст",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Обработчик: занято / доступно",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(messenger_worker_busy)",
"refId": "A",
"legendFormat": "занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(messenger_worker_capacity)",
"refId": "B",
"legendFormat": "всего мест",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "Соединения PostgreSQL",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name) (messenger_db_pool_used)",
"refId": "A",
"legendFormat": "{{service_name}} занято",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum by(service_name) (messenger_db_pool_waiting)",
"refId": "B",
"legendFormat": "{{service_name}} ожидают",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Попытки обработки по результату",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(outcome) (rate(messenger_job_attempts_total[2m]))",
"refId": "A",
"legendFormat": "{{outcome}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "Длительность попытки обработки: p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "От приёма до ready: p50 / p95 / p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "histogram_quantile(0.5,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "A",
"legendFormat": "p50",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "B",
"legendFormat": "p95",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
"refId": "C",
"legendFormat": "p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Only completed publications have a duration. Deadline SLI below also counts unfinished tasks.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "SLO API за 30 минут: цель 99%",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "1 - sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[30m])) / clamp_min(sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\"}[30m])),1)",
"refId": "A",
"legendFormat": "доступность",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "SLO ready за 30 минут: 99% не позже 10 секунд",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(increase(messenger_slo_decisions_total{outcome=\"good\"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)",
"refId": "A",
"legendFormat": "доля вовремя",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "Durable one-time decisions per non-obsolete logical job; unfinished jobs become bad at ten seconds. Synthetic 320x200 profile.",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "CPU процессов, ядра",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum by(service_name)(rate(messenger_process_cpu_seconds_total[1m]))",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Память процессов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "max by(service_name)(messenger_process_memory_bytes)",
"refId": "A",
"legendFormat": "{{service_name}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Буфер экспорта Collector",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "otelcol_exporter_queue_size",
"refId": "A",
"legendFormat": "{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "otelcol_exporter_queue_capacity",
"refId": "B",
"legendFormat": "{{exporter}} capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Ошибки отправки трасс и логов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
"refId": "A",
"legendFormat": "spans/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
"refId": "B",
"legendFormat": "logs/s",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Состояния локальных alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"expr": "ALERTS{alertstate=\"firing\"}",
"refId": "A",
"legendFormat": "{{alertname}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Связанные логи API и обработчика",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"queryType": "range",
"datasource": {
"type": "loki",
"uid": "loki"
}
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending"
},
"description": "Filter structured metadata trace_id in Explore; open the same ID in the Traces datasource."
}
],
"annotations": {
"list": []
}
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Laboratory 6
type: file
disableDeletion: true
editable: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,37 @@
apiVersion: 1
datasources:
- name: Metrics
uid: prometheus
type: prometheus
access: proxy
url: http://127.0.0.1:13909
isDefault: true
editable: false
jsonData:
httpMethod: POST
timeInterval: 5s
- name: Logs
uid: loki
type: loki
access: proxy
url: http://127.0.0.1:13310
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
url: $${__value.raw}
datasourceUid: tempo
- name: Traces
uid: tempo
type: tempo
access: proxy
url: http://127.0.0.1:13320
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
filterByTraceID: true
spanStartTimeShift: -1m
spanEndTimeShift: 1m
+55
View File
@@ -0,0 +1,55 @@
global
log stdout format raw local0
maxconn 512
ssl-default-bind-options ssl-min-ver TLSv1.3 no-tls-tickets
ssl-default-server-options ssl-min-ver TLSv1.3 no-tls-tickets
hard-stop-after 10s
mworker-max-reloads 2
stats socket /tmp/admin.sock mode 600 level admin
defaults
mode http
log global
log-format "http status=%ST duration_ms=%Ta"
timeout connect 1s
timeout client 15s
timeout server 8s
timeout http-request 10s
timeout http-keep-alive 5s
retries 0
option httpclose
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8443 ssl crt /run/identity/current/bundle.pem
use_backend health if { path_beg /health/ }
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
server-template api 1-8 api:8443 ssl verify required ca-file /trust/ca.pem crt /run/identity/current/bundle.pem sni str(api) check-sni api verifyhost api no-ssl-reuse
backend health
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
server-template api 1-8 api:8443 ssl verify required ca-file /trust/ca.pem crt /run/identity/current/bundle.pem sni str(api) check-sni api verifyhost api no-ssl-reuse
frontend internal_metrics
bind 127.0.0.1:8404
http-request use-service prometheus-exporter if { path /metrics }
http-request deny
+14
View File
@@ -0,0 +1,14 @@
disable_mlock = true
ui = false
api_addr = "https://openbao:8200"
storage "file" { path = "/data/storage" }
listener "tcp" {
address = "0.0.0.0:8200"
tls_min_version = "tls13"
tls_cert_file = "/run/identity/current/cert.pem"
tls_key_file = "/run/identity/current/key.pem"
tls_client_ca_file = "/trust/ca.pem"
tls_require_and_verify_client_cert = true
tls_auto_reload = true
tls_auto_reload_interval = "5s"
}
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/api" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/api" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/broker" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/cache" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/grafana" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/lb" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/loki" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/migrate" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/migrate" { capabilities = ["read"] }
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/observer" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/observer" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/openbao" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+11
View File
@@ -0,0 +1,11 @@
path "pki/*" { capabilities = ["create", "read", "update", "delete", "list", "sudo"] }
path "auth/cert/*" { capabilities = ["create", "read", "update", "delete", "list"] }
path "secret/*" { capabilities = ["create", "read", "update", "delete", "list"] }
path "sys/policies/acl/messenger-*" { capabilities = ["create", "read", "update", "delete", "list"] }
path "sys/mounts/pki/tune" { capabilities = ["read", "update", "sudo"] }
path "sys/seal" { capabilities = ["update", "sudo"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/accessors" { capabilities = ["list", "sudo"] }
path "auth/token/lookup-accessor" { capabilities = ["update", "sudo"] }
path "auth/token/revoke-accessor" { capabilities = ["update", "sudo"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/otel" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/postgres" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/prometheus" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/reconciler" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/reconciler" { capabilities = ["read"] }
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/relay" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/relay" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/s3" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+3
View File
@@ -0,0 +1,3 @@
path "pki/issue/tempo" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
+4
View File
@@ -0,0 +1,4 @@
path "pki/issue/worker" { capabilities = ["create", "update"] }
path "auth/token/renew-self" { capabilities = ["update"] }
path "auth/token/lookup-self" { capabilities = ["read"] }
path "secret/data/worker" { capabilities = ["read"] }
+50
View File
@@ -0,0 +1,50 @@
# Диагностика по сохранённой телеметрии
Открой dashboard «Мессенджер: диагностика и SLO», временная зона UTC.
Сопоставляй время оповещения с метриками, трассами и связанными логами.
## API errors
Признак: ApiHighErrorRate, ApiFastBurn или ApiSlowBurn. Числитель доступности:
`sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m]))`.
Знаменатель — тот же backend со всеми code. Health использует другой backend.
Сравни API histogram, DB pool used/waiting и worker busy. В Traces найди
ошибочный server span и вложенный db.postgresql или s3.*. Длинная DB-операция
с ошибкой и занятым пулом при нормальной очереди указывает на БД. Отсутствие
API span при ingress503 может означать отсутствие готовых API. Один5xx не
доказывает причину. Через trace_id найди соответствующий Logs record.
После исправления проверь новые сообщения, спад5xx, resolved и прежние данные.
## Queue age
Признак: QueueLagGrowing, ReadyFastBurn или ReadySlowBurn. Сравни логические
queued jobs PG, oldest job age и worker busy/capacity. Upload202 при росте
ожидающей очереди и отсутствии новых worker spans указывает на обработчик.
Если попытки идут и содержат DB/S3 ошибки, причина может быть в зависимости.
Найди путь upload→producer→consumer→S3/DB. Histogram ready содержит только
завершения, поэтому отдельно проверь deadline SLI для незавершённых jobs.
После восстановления проверь тот же attachment_id, original, спад и resolved.
## Telemetry backend
Проверь backend health, Collector queue и accepted/sent/failed счётчики.
Failed attempts не равны числу уникально потерянных spans: пакет повторяется.
После слива очередей сравни изменение accepted-minus-sent и исходное состояние.
SDK queues traces/logs512records, timeout0.75s; Collector memory limiter192MiB,
batch≤256, очереди logs/traces по32пакета, один sender и finite retry≤10s.
При отказе telemetry может теряться; измеряй память и настоящие бизнес-запросы.
После восстановления найди новую трассу и лог, а не только старый screenshot.
## SLO и окна
Учебный профиль: синтетические PNG320×200, upload не чаще одного каждые3s,
закреплённые CPU/RAM. API цель99%ответов без5xx за30m на ingress. Jobs цель99%
ready≤10s от PG created_at принимающей транзакции до committed_at публикации.
Начальная граница консервативно включает завершение принимающей транзакции.
Однократные durable good/bad решения учитывают failed и незавершённые после10s.
Ручной replay не стирает прежний bad; устаревшие до решения jobs исключены.
Burn rate = доля плохих событий/0.01. Быстрое правило: burn>14.4 в30s и2m,
непрерывно45s. Длительное: burn>6 в2m и5m, непрерывно60s. Короткие окна выбраны
для воспроизводимого стенда, а не универсальной production paging policy.
Измеряются краткий всплеск/длительная деградация/firing/resolved/ложные события.
Точность времени ограничена polling2s, scrape/evaluation5s и экспортом.
+53
View File
@@ -0,0 +1,53 @@
groups:
- name: messenger-slo
interval: 5s
rules:
- alert: ApiHighErrorRate
expr: sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) > 0.05
for: 20s
labels: {severity: warning, signal: api}
annotations:
summary: More than 5 percent business responses are 5xx
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: QueueLagGrowing
expr: max(messenger_oldest_job_age_seconds) > 20 and max(messenger_queue_depth) > 0
for: 10s
labels: {severity: warning, signal: ready}
annotations:
summary: Oldest unfinished attachment exceeds 20 seconds
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ApiFastBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[30s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[30s])),0.001) / 0.01 > 14.4) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: api, burn: fast}
annotations:
summary: API burns the one-percent error budget in both fast windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ApiSlowBurn
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 6) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: api, burn: slow}
annotations:
summary: API burns the error budget across two longer windows
runbook: infra/otel/RUNBOOK.md#api-errors
- alert: ReadyFastBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[30s])) / clamp_min(sum(rate(messenger_slo_decisions_total[30s])),0.001) / 0.01 > 14.4) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 14.4)
for: 45s
labels: {severity: critical, signal: ready, burn: fast}
annotations:
summary: Attachment deadline SLO burns the error budget
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: ReadySlowBurn
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 6) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[5m])) / clamp_min(sum(rate(messenger_slo_decisions_total[5m])),0.001) / 0.01 > 6)
for: 60s
labels: {severity: critical, signal: ready, burn: slow}
annotations:
summary: Sustained misses of the attachment ten-second deadline
runbook: infra/otel/RUNBOOK.md#queue-age
- alert: CollectorUnavailable
expr: up{job="collector"} == 0
for: 15s
labels: {severity: warning, signal: telemetry}
annotations:
summary: Collector self metrics are unavailable
runbook: infra/otel/RUNBOOK.md#telemetry-backend
+88
View File
@@ -0,0 +1,88 @@
receivers:
otlp:
protocols:
http:
endpoint: 127.0.0.1:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 127.0.0.1:8889
metric_expiration: 15s
resource_constant_labels:
included:
- service.name
- service.instance.id
- service.version
- deployment.environment.name
otlp_http/tempo:
endpoint: http://127.0.0.1:14319
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
otlp_http/loki:
endpoint: http://127.0.0.1:13310/otlp
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 1
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 10s
extensions:
health_check:
endpoint: 127.0.0.1:13133
service:
extensions:
- health_check
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 127.0.0.1
port: 8888
pipelines:
traces:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/tempo
logs:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/loki
metrics:
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- prometheus
+44
View File
@@ -0,0 +1,44 @@
auth_enabled: false
server:
http_listen_port: 3100
log_level: warn
http_listen_address: 127.0.0.1
grpc_listen_address: 127.0.0.1
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2026-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
ingestion_rate_mb: 4
ingestion_burst_size_mb: 8
volume_enabled: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes:
- service.name
- action: structured_metadata
attributes:
- service.instance.id
- service.version
- deployment.environment.name
analytics:
reporting_enabled: false
+30
View File
@@ -0,0 +1,30 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: application_otlp
static_configs:
- targets:
- 127.0.0.1:18889
- job_name: collector
static_configs:
- targets:
- 127.0.0.1:18888
- job_name: ingress
static_configs:
- targets:
- 127.0.0.1:18404
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: tempo
static_configs:
- targets:
- 127.0.0.1:13200
- job_name: loki
static_configs:
- targets:
- 127.0.0.1:13100
+22
View File
@@ -0,0 +1,22 @@
target: all
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
http_listen_address: 127.0.0.1
grpc_listen_address: 127.0.0.1
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 127.0.0.1:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+7
View File
@@ -0,0 +1,7 @@
{
"otel": "otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea",
"prometheus": "prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
"loki": "grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
"tempo": "grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044",
"grafana": "grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
}
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
root_user=$(cat /run/secrets/s3_root_user)
root_password=$(cat /run/secrets/s3_root_password)
api_key=$(cat /run/secrets/s3_access_key)
api_secret=$(cat /run/secrets/s3_secret_key)
# Credentials are passed only inside this isolated bootstrap container.
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
mc mb --ignore-existing local/messenger >/dev/null 2>&1
mc anonymous set none local/messenger >/dev/null 2>&1
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
printf '%s\n' 'Private bucket and scoped application policy ready'
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
export MINIO_BROWSER=off
exec minio server /data --address 127.0.0.1:9000 --console-address 127.0.0.1:9001
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger/objects/*"]
}
]
}
+49
View File
@@ -0,0 +1,49 @@
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
for attempt in 1 2 3 4; do \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
test "$attempt" -lt 4 || exit 1; sleep 1; \
done && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
for attempt in 1 2 3; do \
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
test "$attempt" -lt 3 || exit 1; sleep 1; \
done
FROM toolchain AS mc-build
WORKDIR /source
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
RUN mkdir /data && chown 10001:10001 /data
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
COPY infra/s3-policy.json /policy.json
USER 10001:10001
ENV MC_CONFIG_DIR=/tmp/mc
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
+17
View File
@@ -0,0 +1,17 @@
{
"role": "api",
"dns": [
"api",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "broker",
"dns": [
"broker"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "cache",
"dns": [
"cache"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "grafana",
"dns": [
"grafana",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"ip": [
"127.0.0.1"
],
"bootstrap": "/bootstrap",
"kv": false
}
+21
View File
@@ -0,0 +1,21 @@
{
"role": "lb",
"dns": [
"lb",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"ip": [
"127.0.0.1"
],
"bootstrap": "/bootstrap",
"kv": false,
"reload_pid_one": true
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "loki",
"dns": [
"loki"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "migrate",
"dns": [
"migrate"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "observer",
"dns": [
"observer"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "openbao",
"dns": [
"openbao",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"ip": [
"127.0.0.1"
],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "otel",
"dns": [
"otel"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "postgres",
"dns": [
"postgres"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "prometheus",
"dns": [
"prometheus"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "reconciler",
"dns": [
"reconciler"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "relay",
"dns": [
"relay"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "s3",
"dns": [
"s3",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"ip": [
"127.0.0.1"
],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "tempo",
"dns": [
"tempo"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": false
}
+16
View File
@@ -0,0 +1,16 @@
{
"role": "worker",
"dns": [
"worker"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": false,
"servers": [],
"outbounds": [],
"bootstrap": "/bootstrap",
"kv": true
}
+33
View File
@@ -0,0 +1,33 @@
{
"role": "api",
"dns": [
"api",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:8443",
"target": "127.0.0.1:8080",
"protocol": "http",
"allow": {
"spiffe://messenger/lb": [
"* /"
]
},
"public": false
}
],
"outbounds": [
{
"listen": "127.0.0.1:14318",
"target": "https://otel:4438",
"server_identity": "spiffe://messenger/otel"
}
]
}
+25
View File
@@ -0,0 +1,25 @@
{
"role": "broker",
"dns": [
"broker"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:6380",
"target": "127.0.0.1:6379",
"protocol": "tcp",
"allow": {
"spiffe://messenger/relay": [],
"spiffe://messenger/worker": [],
"spiffe://messenger/operator": []
}
}
],
"outbounds": []
}
+24
View File
@@ -0,0 +1,24 @@
{
"role": "cache",
"dns": [
"cache"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:6380",
"target": "127.0.0.1:6379",
"protocol": "tcp",
"allow": {
"spiffe://messenger/api": [],
"spiffe://messenger/operator": []
}
}
],
"outbounds": []
}
+631
View File
@@ -0,0 +1,631 @@
// A bounded TLS boundary; cleartext targets must bind only to loopback.
package main
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
)
type Boundary struct {
Listen string `json:"listen"`
Target string `json:"target"`
Protocol string `json:"protocol"`
// Identity -> allowed HTTP method/path prefixes; an empty list denies HTTP.
Allow map[string][]string `json:"allow"`
Public bool `json:"public"`
}
type Outbound struct {
Listen string `json:"listen"`
Target string `json:"target"`
Identity string `json:"server_identity"`
}
type Config struct {
Outbounds []Outbound `json:"outbounds"`
ServeOnly bool `json:"serve_only"`
ReloadPIDOne bool `json:"reload_pid_one"`
Role string `json:"role"`
DNS []string `json:"dns"`
IP []string `json:"ip"`
Bao string `json:"bao"`
Trust string `json:"trust"`
Bootstrap string `json:"bootstrap"`
Identity string `json:"identity"`
Revoked string `json:"revoked"`
TTL string `json:"ttl"`
KV bool `json:"kv"`
Servers []Boundary `json:"servers"`
}
type Agent struct {
c Config
mu sync.Mutex
token string
tokenUntil time.Time
}
func event(name string) { log.Print("{\"event\":\"" + name + "\"}") }
func trust(c Config) (*x509.CertPool, error) {
raw, e := os.ReadFile(c.Trust)
if e != nil {
return nil, e
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(raw) {
return nil, errors.New("invalid trust bundle")
}
return pool, nil
}
func material(c Config) (tls.Certificate, error) {
base := filepath.Join(c.Identity, "current")
if real, e := filepath.EvalSymlinks(base); e == nil {
base = real
} else {
base = c.Bootstrap
}
return tls.LoadX509KeyPair(filepath.Join(base, "cert.pem"), filepath.Join(base, "key.pem"))
}
func revoked(c Config, cert *x509.Certificate) bool {
raw, e := os.ReadFile(c.Revoked)
if e != nil {
return true
} // missing policy fails closed
var entries map[string]bool
if json.Unmarshal(raw, &entries) != nil {
return true
}
return entries[strings.ToLower(cert.SerialNumber.Text(16))]
}
func peer(c Config, state tls.ConnectionState, b Boundary) (string, error) {
if b.Public {
return "", nil
}
if len(state.VerifiedChains) == 0 || len(state.PeerCertificates) == 0 {
return "", errors.New("client certificate required")
}
cert := state.PeerCertificates[0]
roots, err := trust(c)
if err != nil {
return "", err
}
intermediates := x509.NewCertPool()
for _, item := range state.PeerCertificates[1:] {
intermediates.AddCert(item)
}
if _, err = cert.Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
return "", errors.New("current trust denies client")
}
if time.Now().Before(cert.NotBefore) || !time.Now().Before(cert.NotAfter) || revoked(c, cert) {
return "", errors.New("client certificate unavailable")
}
for _, identity := range cert.URIs {
if _, ok := b.Allow[identity.String()]; ok {
return identity.String(), nil
}
}
return "", errors.New("client identity denied")
}
func serverTLS(c Config, b Boundary) *tls.Config {
return &tls.Config{MinVersion: tls.VersionTLS13, SessionTicketsDisabled: true, GetConfigForClient: func(_ *tls.ClientHelloInfo) (*tls.Config, error) {
cert, e := material(c)
if e != nil {
return nil, e
}
roots, e := trust(c)
if e != nil {
return nil, e
}
conf := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}, ClientCAs: roots, SessionTicketsDisabled: true}
if !b.Public {
conf.ClientAuth = tls.RequireAndVerifyClientCert
conf.VerifyConnection = func(s tls.ConnectionState) error { _, e := peer(c, s, b); return e }
}
return conf, nil
}}
}
func atomicFile(path string, raw []byte) error {
temp := path + ".new"
if e := os.WriteFile(temp, raw, 0600); e != nil {
return e
}
return os.Rename(temp, path)
}
func (a *Agent) request(method, path string, body any, authenticated bool) (map[string]any, error) {
cert, e := material(a.c)
if e != nil {
return nil, e
}
roots, e := trust(a.c)
if e != nil {
return nil, e
}
transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{cert}}, DisableKeepAlives: true, ResponseHeaderTimeout: 4 * time.Second}
defer transport.CloseIdleConnections()
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
var encoded []byte
if body != nil {
encoded, e = json.Marshal(body)
if e != nil {
return nil, e
}
}
request, e := http.NewRequest(method, a.c.Bao+"/v1/"+path, bytes.NewReader(encoded))
if e != nil {
return nil, e
}
request.Header.Set("Content-Type", "application/json")
if authenticated {
request.Header.Set("X-Vault-Token", a.token)
}
response, e := client.Do(request)
if e != nil {
return nil, errors.New("identity service unavailable")
}
defer response.Body.Close()
if response.StatusCode >= 300 {
return nil, fmt.Errorf("identity service status %d", response.StatusCode)
}
var data map[string]any
e = json.NewDecoder(io.LimitReader(response.Body, 1024*1024)).Decode(&data)
return data, e
}
func (a *Agent) refresh() error {
a.mu.Lock()
defer a.mu.Unlock()
now := time.Now()
if a.token != "" && time.Until(a.tokenUntil) < 95*time.Second {
result, err := a.request("POST", "auth/token/renew-self", map[string]string{"increment": "120s"}, true)
if err == nil {
auth, ok := result["auth"].(map[string]any)
if !ok {
return errors.New("invalid renewal response")
}
ttl, ok := auth["lease_duration"].(float64)
if !ok || ttl < 1 {
return errors.New("invalid renewed lease")
}
a.tokenUntil = now.Add(time.Duration(ttl * float64(time.Second)))
event("identity.token_renewed")
} else {
a.token = ""
}
}
if a.token == "" {
// The exact CN+URI constraints select the correct role. Omitting name
// permits old/new CA roles to coexist safely during trust rollover.
result, err := a.request("POST", "auth/cert/login", map[string]string{}, false)
if err != nil {
return err
}
auth, ok := result["auth"].(map[string]any)
if !ok {
return errors.New("invalid auth response")
}
a.token, ok = auth["client_token"].(string)
if !ok {
return errors.New("missing service token")
}
ttl, ok := auth["lease_duration"].(float64)
if !ok || ttl < 1 {
return errors.New("invalid service lease")
}
a.tokenUntil = now.Add(time.Duration(ttl * float64(time.Second)))
event("identity.authenticated")
}
cert, e := material(a.c)
_, currentErr := os.Stat(filepath.Join(a.c.Identity, "current"))
_, initializedErr := os.Stat(filepath.Join(a.c.Identity, "agent.json"))
renew := initializedErr != nil || currentErr != nil || e != nil || cert.Leaf == nil || time.Until(cert.Leaf.NotAfter) < 80*time.Second
if renew {
request := map[string]any{"common_name": a.c.Role, "alt_names": strings.Join(a.c.DNS, ","), "uri_sans": "spiffe://messenger/" + a.c.Role, "ttl": a.c.TTL}
if len(a.c.IP) > 0 {
request["ip_sans"] = strings.Join(a.c.IP, ",")
}
result, e := a.request("POST", "pki/issue/"+a.c.Role, request, true)
if e != nil {
return e
}
data, ok := result["data"].(map[string]any)
if !ok {
return errors.New("invalid PKI response")
}
crt, ok := data["certificate"].(string)
if !ok {
return errors.New("missing certificate")
}
key, ok := data["private_key"].(string)
if !ok {
return errors.New("missing private key")
}
if chain, ok := data["ca_chain"].([]any); ok {
for _, item := range chain {
crt += "\n" + item.(string)
}
}
parsed, e := tls.X509KeyPair([]byte(crt), []byte(key))
if e != nil {
return e
}
dir, e := os.MkdirTemp(a.c.Identity, "generation-")
if e != nil {
return e
}
if e = os.WriteFile(filepath.Join(dir, "cert.pem"), []byte(crt), 0600); e != nil {
return e
}
if e = os.WriteFile(filepath.Join(dir, "key.pem"), []byte(key), 0600); e != nil {
return e
}
if e = os.WriteFile(filepath.Join(dir, "bundle.pem"), []byte(crt+"\n"+key), 0600); e != nil {
return e
}
link := filepath.Join(a.c.Identity, "next")
os.Remove(link)
if e = os.Symlink(filepath.Base(dir), link); e != nil {
return e
}
if e = os.Rename(link, filepath.Join(a.c.Identity, "current")); e != nil {
return e
}
// Retain only the current and immediately previous generations, bounding tmpfs.
entries, _ := os.ReadDir(a.c.Identity)
for _, entry := range entries {
if entry.IsDir() && strings.HasPrefix(entry.Name(), "generation-") && entry.Name() != filepath.Base(dir) {
info, _ := entry.Info()
if info != nil && now.Sub(info.ModTime()) > 3*time.Minute {
os.RemoveAll(filepath.Join(a.c.Identity, entry.Name()))
}
}
}
if parsed.Leaf == nil {
return errors.New("unparsed certificate")
}
event("identity.certificate_issued")
if a.c.ReloadPIDOne {
if e = syscall.Kill(1, syscall.SIGUSR2); e != nil {
return errors.New("certificate reload failed")
}
}
}
if a.c.KV {
result, e := a.request("GET", "secret/data/"+a.c.Role, nil, true)
if e != nil {
return e
}
outer, ok := result["data"].(map[string]any)
if !ok {
return errors.New("invalid secret response")
}
values, ok := outer["data"].(map[string]any)
if !ok {
return errors.New("invalid secret values")
}
expiry := now.Add(60 * time.Second)
if a.tokenUntil.Before(expiry) {
expiry = a.tokenUntil
}
data, e := json.Marshal(map[string]any{"fetched_at": now.Unix(), "valid_until": expiry.Unix(), "values": values})
if e != nil {
return e
}
if e = atomicFile(filepath.Join(a.c.Identity, "secrets.json"), data); e != nil {
return e
}
}
return atomicFile(filepath.Join(a.c.Identity, "agent.json"), []byte(`{"refreshed_at":`+strconv.FormatInt(now.Unix(), 10)+`}`))
}
type boundedListener struct {
net.Listener
slots chan struct{}
}
type boundedConn struct {
net.Conn
once sync.Once
release func()
}
func (c *boundedConn) Close() error { e := c.Conn.Close(); c.once.Do(c.release); return e }
func (l *boundedListener) Accept() (net.Conn, error) {
for {
c, e := l.Listener.Accept()
if e != nil {
return nil, e
}
select {
case l.slots <- struct{}{}:
return &boundedConn{Conn: c, release: func() { <-l.slots }}, nil
default:
c.Close()
}
}
}
func runHTTP(c Config, b Boundary) error {
target, e := url.Parse("http://" + b.Target)
if e != nil {
return e
}
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.Transport = &http.Transport{MaxIdleConns: 32, MaxIdleConnsPerHost: 16, ResponseHeaderTimeout: 8 * time.Second, IdleConnTimeout: 30 * time.Second}
proxy.ErrorLog = log.New(io.Discard, "", 0)
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
w.Header().Set("Connection", "close")
http.Error(w, "Protected service unavailable", 503)
}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Force a fresh handshake after each response: no indefinitely cached
// certificate identity across rollover, expiry or serial revocation.
w.Header().Set("Connection", "close")
identity, e := peer(c, *r.TLS, b)
if e != nil {
w.Header().Set("Connection", "close")
http.Error(w, "Client identity denied", 403)
return
}
allowed := b.Public
for _, rule := range b.Allow[identity] {
parts := strings.SplitN(rule, " ", 2)
if len(parts) == 2 && (parts[0] == "*" || parts[0] == r.Method) && strings.HasPrefix(r.URL.Path, parts[1]) {
allowed = true
}
}
if !allowed {
http.Error(w, "Operation denied", 403)
return
}
r.Header.Del("X-Forwarded-Client-Cert")
r.Header.Del("X-TLS-Identity")
proxy.ServeHTTP(w, r)
})
listener, e := net.Listen("tcp", b.Listen)
if e != nil {
return e
}
listener = &boundedListener{Listener: listener, slots: make(chan struct{}, 128)}
server := &http.Server{Handler: handler, TLSConfig: serverTLS(c, b), ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 32768, ErrorLog: log.New(io.Discard, "", 0)}
return server.Serve(tls.NewListener(listener, server.TLSConfig))
}
// Native clients use a private loopback endpoint when their certificate
// reload behavior cannot meet the short credential lifetime. Every network
// connection is verified mTLS; configuration cannot publish this HTTP hop.
func runOutbound(c Config, b Outbound) error {
target, e := url.Parse(b.Target)
if e != nil || target.Scheme != "https" || target.Hostname() == "" {
return errors.New("outbound TLS target required")
}
host, _, e := net.SplitHostPort(b.Listen)
if e != nil || host != "127.0.0.1" {
return errors.New("outbound listener must be loopback")
}
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.Transport = &http.Transport{
DisableKeepAlives: true, MaxConnsPerHost: 32, ResponseHeaderTimeout: 8 * time.Second,
DialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) {
cert, e := material(c)
if e != nil {
return nil, e
}
roots, e := trust(c)
if e != nil {
return nil, e
}
config := &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{cert}, ServerName: target.Hostname(), VerifyConnection: func(state tls.ConnectionState) error {
if len(state.VerifiedChains) == 0 || len(state.PeerCertificates) == 0 {
return errors.New("unverified protected server")
}
leaf := state.PeerCertificates[0]
if revoked(c, leaf) {
return errors.New("protected server revoked")
}
if b.Identity != "" {
found := false
for _, identity := range leaf.URIs {
if identity.String() == b.Identity {
found = true
}
}
if !found {
return errors.New("protected server identity denied")
}
}
return nil
}}
dialer := tls.Dialer{NetDialer: &net.Dialer{Timeout: 2 * time.Second}, Config: config}
return dialer.DialContext(ctx, network, address)
},
}
proxy.ErrorLog = log.New(io.Discard, "", 0)
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
w.Header().Set("Connection", "close")
http.Error(w, "Protected destination unavailable", 503)
}
listener, e := net.Listen("tcp", b.Listen)
if e != nil {
return e
}
listener = &boundedListener{Listener: listener, slots: make(chan struct{}, 64)}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Connection", "close")
proxy.ServeHTTP(w, r)
})
server := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second, MaxHeaderBytes: 32768, ErrorLog: log.New(io.Discard, "", 0)}
return server.Serve(listener)
}
func runTCP(c Config, b Boundary) error {
listener, e := net.Listen("tcp", b.Listen)
if e != nil {
return e
}
slots := make(chan struct{}, 128)
for {
raw, e := listener.Accept()
if e != nil {
return e
}
select {
case slots <- struct{}{}:
default:
raw.Close()
continue
}
go func() {
defer func() { <-slots }()
defer raw.Close()
raw.SetDeadline(time.Now().Add(5 * time.Second))
if b.Protocol == "postgres" {
request := make([]byte, 8)
if _, e := io.ReadFull(raw, request); e != nil {
return
}
if binary.BigEndian.Uint32(request[:4]) != 8 || binary.BigEndian.Uint32(request[4:]) != 80877103 {
return
}
if _, e := raw.Write([]byte{'S'}); e != nil {
return
}
}
client := tls.Server(raw, serverTLS(c, b))
if e := client.Handshake(); e != nil {
return
}
target, e := net.DialTimeout("tcp", b.Target, 2*time.Second)
if e != nil {
return
}
defer target.Close()
// Bound a previously authenticated connection even after expiry/revocation.
deadline := time.Now().Add(45 * time.Second)
state := client.ConnectionState()
if len(state.PeerCertificates) > 0 && state.PeerCertificates[0].NotAfter.Before(deadline) {
deadline = state.PeerCertificates[0].NotAfter
}
client.SetDeadline(deadline)
target.SetDeadline(deadline)
done := make(chan struct{}, 1)
go func() { io.Copy(target, client); done <- struct{}{} }()
io.Copy(client, target)
client.Close()
target.Close()
<-done
}()
}
}
func main() {
if len(os.Args) != 2 {
event("gateway.invalid_configuration")
os.Exit(2)
}
raw, e := os.ReadFile(os.Args[1])
if e != nil {
event("gateway.invalid_configuration")
os.Exit(2)
}
var c Config
if json.Unmarshal(raw, &c) != nil || c.Role == "" {
event("gateway.invalid_configuration")
os.Exit(2)
}
if c.TTL == "" {
c.TTL = "120s"
}
if e = os.MkdirAll(c.Identity, 0700); e != nil {
event("gateway.identity_directory_unavailable")
os.Exit(2)
}
for _, b := range c.Servers {
host, _, err := net.SplitHostPort(b.Target)
if err != nil || (host != "127.0.0.1" && host != "[::1]" && host != "::1") {
event("gateway.nonlocal_target_denied")
os.Exit(2)
}
}
agent := &Agent{c: c}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
// Bootstrap failures never start a cleartext fallback listener.
if !c.ServeOnly {
for {
if e = agent.refresh(); e == nil {
break
}
event("identity.refresh_failed")
select {
case <-ctx.Done():
return
case <-time.After(3 * time.Second):
}
}
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if agent.refresh() != nil {
event("identity.refresh_failed")
}
}
}
}()
} else {
for {
cert, err := material(c)
if err == nil && cert.Leaf != nil && time.Now().Before(cert.Leaf.NotAfter) {
break
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}
for _, boundary := range c.Servers {
go func(b Boundary) {
var e error
if b.Protocol == "http" {
e = runHTTP(c, b)
} else {
e = runTCP(c, b)
}
if e != nil {
event("gateway.listener_failed")
stop()
}
}(boundary)
}
for _, outbound := range c.Outbounds {
go func(b Outbound) {
if e := runOutbound(c, b); e != nil {
event("gateway.outbound_failed")
stop()
}
}(outbound)
}
event("gateway.ready")
<-ctx.Done()
}
+122
View File
@@ -0,0 +1,122 @@
package main
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"math/big"
"net/url"
"os"
"path/filepath"
"testing"
"time"
)
type authority struct {
cert *x509.Certificate
key *ecdsa.PrivateKey
pem []byte
}
func root(t *testing.T) authority {
t.Helper()
key, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if e != nil {
t.Fatal(e)
}
template := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "Synthetic test CA"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
der, e := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if e != nil {
t.Fatal(e)
}
cert, e := x509.ParseCertificate(der)
if e != nil {
t.Fatal(e)
}
return authority{cert, key, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})}
}
func client(t *testing.T, ca authority, role string, expired bool) *x509.Certificate {
t.Helper()
key, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if e != nil {
t.Fatal(e)
}
identity, _ := url.Parse("spiffe://messenger/" + role)
expiry := time.Now().Add(time.Minute)
if expired {
expiry = time.Now().Add(-time.Minute)
}
template := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: role}, NotBefore: time.Now().Add(-time.Hour), NotAfter: expiry, URIs: []*url.URL{identity}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, KeyUsage: x509.KeyUsageDigitalSignature}
der, e := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key)
if e != nil {
t.Fatal(e)
}
cert, e := x509.ParseCertificate(der)
if e != nil {
t.Fatal(e)
}
return cert
}
func TestClientAuthorizationUsesCurrentChainIdentityExpiryAndRevocation(t *testing.T) {
ca := root(t)
foreign := root(t)
directory := t.TempDir()
conf := Config{Trust: filepath.Join(directory, "ca.pem"), Revoked: filepath.Join(directory, "revoked.json")}
if e := os.WriteFile(conf.Trust, ca.pem, 0600); e != nil {
t.Fatal(e)
}
if e := os.WriteFile(conf.Revoked, []byte(`{}`), 0600); e != nil {
t.Fatal(e)
}
boundary := Boundary{Allow: map[string][]string{"spiffe://messenger/lb": {"* /"}}}
good := client(t, ca, "lb", false)
state := func(leaf *x509.Certificate, issuer authority) tls.ConnectionState {
return tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf, issuer.cert}, VerifiedChains: [][]*x509.Certificate{{leaf, issuer.cert}}}
}
cases := []struct {
name string
state tls.ConnectionState
allowed bool
}{
{"authorized", state(good, ca), true},
{"no_certificate", tls.ConnectionState{}, false},
{"trusted_but_wrong_role", state(client(t, ca, "worker", false), ca), false},
{"foreign_authority", state(client(t, foreign, "lb", false), foreign), false},
{"expired", state(client(t, ca, "lb", true), ca), false},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
_, e := peer(conf, test.state, boundary)
if (e == nil) != test.allowed {
t.Fatalf("authorization mismatch: allowed=%v error=%v", test.allowed, e)
}
})
}
raw, _ := json.Marshal(map[string]bool{good.SerialNumber.Text(16): true})
if e := os.WriteFile(conf.Revoked, raw, 0600); e != nil {
t.Fatal(e)
}
if _, e := peer(conf, state(good, ca), boundary); e == nil {
t.Fatal("revoked client accepted on previously authenticated connection")
}
if e := os.WriteFile(conf.Revoked, []byte(`{}`), 0600); e != nil {
t.Fatal(e)
}
if e := os.WriteFile(conf.Trust, foreign.pem, 0600); e != nil {
t.Fatal(e)
}
if _, e := peer(conf, state(good, ca), boundary); e == nil {
t.Fatal("retired authority accepted from cached verified chain")
}
if e := os.Remove(conf.Revoked); e != nil {
t.Fatal(e)
}
if _, e := peer(conf, state(good, ca), boundary); e == nil {
t.Fatal("missing revocation policy did not fail closed")
}
}
+42
View File
@@ -0,0 +1,42 @@
{
"role": "grafana",
"dns": [
"grafana",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:3443",
"target": "127.0.0.1:3000",
"protocol": "http",
"allow": {},
"public": true
}
],
"outbounds": [
{
"listen": "127.0.0.1:13909",
"target": "https://prometheus:9900",
"server_identity": "spiffe://messenger/prometheus"
},
{
"listen": "127.0.0.1:13310",
"target": "https://loki:3310",
"server_identity": "spiffe://messenger/loki"
},
{
"listen": "127.0.0.1:13320",
"target": "https://tempo:3320",
"server_identity": "spiffe://messenger/tempo"
}
],
"ip": [
"127.0.0.1"
]
}
+33
View File
@@ -0,0 +1,33 @@
{
"role": "lb",
"dns": [
"lb",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:18404",
"target": "127.0.0.1:8404",
"protocol": "http",
"allow": {
"spiffe://messenger/prometheus": [
"GET /metrics"
],
"spiffe://messenger/operator": [
"GET /metrics"
]
},
"public": false
}
],
"outbounds": [],
"ip": [
"127.0.0.1"
]
}
+36
View File
@@ -0,0 +1,36 @@
{
"role": "loki",
"dns": [
"loki"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:3310",
"target": "127.0.0.1:3100",
"protocol": "http",
"allow": {
"spiffe://messenger/grafana": [
"GET /",
"POST /api/"
],
"spiffe://messenger/prometheus": [
"GET /metrics"
],
"spiffe://messenger/operator": [
"GET /"
],
"spiffe://messenger/otel": [
"POST /otlp/"
]
},
"public": false
}
],
"outbounds": []
}
+14
View File
@@ -0,0 +1,14 @@
{
"role": "migrate",
"dns": [
"migrate"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [],
"outbounds": []
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "observer",
"dns": [
"observer"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [],
"outbounds": [
{
"listen": "127.0.0.1:14318",
"target": "https://otel:4438",
"server_identity": "spiffe://messenger/otel"
}
]
}
+88
View File
@@ -0,0 +1,88 @@
{
"role": "otel",
"dns": [
"otel"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:4438",
"target": "127.0.0.1:4318",
"protocol": "http",
"allow": {
"spiffe://messenger/api": [
"POST /v1/"
],
"spiffe://messenger/worker": [
"POST /v1/"
],
"spiffe://messenger/relay": [
"POST /v1/"
],
"spiffe://messenger/observer": [
"POST /v1/"
],
"spiffe://messenger/operator": [
"POST /v1/"
]
},
"public": false
},
{
"listen": "0.0.0.0:18889",
"target": "127.0.0.1:8889",
"protocol": "http",
"allow": {
"spiffe://messenger/prometheus": [
"GET /metrics"
],
"spiffe://messenger/operator": [
"GET /metrics"
]
},
"public": false
},
{
"listen": "0.0.0.0:18888",
"target": "127.0.0.1:8888",
"protocol": "http",
"allow": {
"spiffe://messenger/prometheus": [
"GET /metrics"
],
"spiffe://messenger/operator": [
"GET /metrics"
]
},
"public": false
},
{
"listen": "0.0.0.0:13333",
"target": "127.0.0.1:13133",
"protocol": "http",
"allow": {
"spiffe://messenger/operator": [
"GET /"
]
},
"public": false
}
],
"outbounds": [
{
"listen": "127.0.0.1:13310",
"target": "https://loki:3310",
"server_identity": "spiffe://messenger/loki"
},
{
"listen": "127.0.0.1:14319",
"target": "https://tempo:4438",
"server_identity": "spiffe://messenger/tempo"
}
]
}
+29
View File
@@ -0,0 +1,29 @@
{
"role": "postgres",
"dns": [
"postgres"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:5443",
"target": "127.0.0.1:5432",
"protocol": "postgres",
"allow": {
"spiffe://messenger/api": [],
"spiffe://messenger/worker": [],
"spiffe://messenger/relay": [],
"spiffe://messenger/observer": [],
"spiffe://messenger/reconciler": [],
"spiffe://messenger/migrate": [],
"spiffe://messenger/operator": []
}
}
],
"outbounds": []
}
+56
View File
@@ -0,0 +1,56 @@
{
"role": "prometheus",
"dns": [
"prometheus"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:9900",
"target": "127.0.0.1:9090",
"protocol": "http",
"allow": {
"spiffe://messenger/grafana": [
"GET /api/",
"POST /api/v1/query"
],
"spiffe://messenger/operator": [
"GET /"
]
},
"public": false
}
],
"outbounds": [
{
"listen": "127.0.0.1:18889",
"target": "https://otel:18889",
"server_identity": "spiffe://messenger/otel"
},
{
"listen": "127.0.0.1:18888",
"target": "https://otel:18888",
"server_identity": "spiffe://messenger/otel"
},
{
"listen": "127.0.0.1:18404",
"target": "https://lb:18404",
"server_identity": "spiffe://messenger/lb"
},
{
"listen": "127.0.0.1:13200",
"target": "https://tempo:3320",
"server_identity": "spiffe://messenger/tempo"
},
{
"listen": "127.0.0.1:13100",
"target": "https://loki:3310",
"server_identity": "spiffe://messenger/loki"
}
]
}
+14
View File
@@ -0,0 +1,14 @@
{
"role": "reconciler",
"dns": [
"reconciler"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [],
"outbounds": []
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "relay",
"dns": [
"relay"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [],
"outbounds": [
{
"listen": "127.0.0.1:14318",
"target": "https://otel:4438",
"server_identity": "spiffe://messenger/otel"
}
]
}
+39
View File
@@ -0,0 +1,39 @@
{
"role": "s3",
"dns": [
"s3",
"localhost"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:9443",
"target": "127.0.0.1:9000",
"protocol": "http",
"allow": {
"spiffe://messenger/api": [
"* /"
],
"spiffe://messenger/worker": [
"* /"
],
"spiffe://messenger/reconciler": [
"* /"
],
"spiffe://messenger/operator": [
"* /"
]
},
"public": false
}
],
"outbounds": [],
"ip": [
"127.0.0.1"
]
}
+47
View File
@@ -0,0 +1,47 @@
{
"role": "tempo",
"dns": [
"tempo"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [
{
"listen": "0.0.0.0:3320",
"target": "127.0.0.1:3200",
"protocol": "http",
"allow": {
"spiffe://messenger/grafana": [
"GET /",
"POST /api/"
],
"spiffe://messenger/prometheus": [
"GET /metrics"
],
"spiffe://messenger/operator": [
"GET /"
]
},
"public": false
},
{
"listen": "0.0.0.0:4438",
"target": "127.0.0.1:4318",
"protocol": "http",
"allow": {
"spiffe://messenger/otel": [
"POST /v1/traces"
],
"spiffe://messenger/operator": [
"POST /v1/traces"
]
},
"public": false
}
],
"outbounds": []
}
+20
View File
@@ -0,0 +1,20 @@
{
"role": "worker",
"dns": [
"worker"
],
"bao": "https://openbao:8200",
"trust": "/trust/ca.pem",
"revoked": "/trust/revoked.json",
"identity": "/run/identity",
"ttl": "120s",
"serve_only": true,
"servers": [],
"outbounds": [
{
"listen": "127.0.0.1:14318",
"target": "https://otel:4438",
"server_identity": "spiffe://messenger/otel"
}
]
}
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id, user_id),
UNIQUE (chat_id, position)
);
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
+32
View File
@@ -0,0 +1,32 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD UNIQUE(chat_id,id);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL,
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
error_code text,
uploader_instance text NOT NULL,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(chat_id,id),
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
);
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
PRIMARY KEY(message_id,attachment_id),
UNIQUE(message_id,position),
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
);
+54
View File
@@ -0,0 +1,54 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
ALTER TABLE attachments ADD COLUMN published_version integer;
CREATE TABLE upload_reservations (
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
);
CREATE TABLE jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
CREATE TABLE outbox (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz
);
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
size_bytes bigint NOT NULL, sha256 text NOT NULL,
width integer, height integer,
PRIMARY KEY(attachment_id,pipeline_version,name)
);
CREATE TABLE effect_publications (
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY(attachment_id,pipeline_version)
);
CREATE TABLE pipeline_audit (
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
operator text NOT NULL, reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
CREATE TABLE pipeline_faults (
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
);
CREATE TABLE pipeline_events (
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+10
View File
@@ -0,0 +1,10 @@
ALTER TABLE jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
ALTER TABLE jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
CREATE TABLE job_slo_observations (
job_id uuid PRIMARY KEY REFERENCES jobs(id),
outcome text NOT NULL CHECK(outcome IN ('good','bad')),
decided_at timestamptz NOT NULL DEFAULT clock_timestamp(),
histogram_epoch uuid
);
CREATE INDEX job_slo_epoch_idx ON job_slo_observations(histogram_epoch);
CREATE INDEX effect_publications_time_idx ON effect_publications(committed_at,job_id);
+32
View File
@@ -0,0 +1,32 @@
-- Only the owner can touch buckets. Callers get a fixed, serialized function.
CREATE TABLE login_buckets (
bucket text PRIMARY KEY, tokens double precision NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX login_buckets_expiry_idx ON login_buckets(updated_at);
CREATE FUNCTION login_wait(account text) RETURNS integer
LANGUAGE plpgsql SECURITY DEFINER SET search_path = public, pg_temp AS $$
DECLARE
moment timestamptz := clock_timestamp();
global_tokens double precision; account_tokens double precision;
BEGIN
IF account !~ '^[a-f0-9]{64}$' THEN RAISE EXCEPTION 'Invalid limiter key'; END IF;
INSERT INTO login_buckets VALUES ('global',8,moment) ON CONFLICT DO NOTHING;
SELECT least(8.0,tokens+extract(epoch FROM moment-updated_at)*2.0)
INTO global_tokens FROM login_buckets WHERE bucket='global' FOR UPDATE;
-- Reject before creating arbitrary account buckets: a name flood cannot
-- grow state faster than the two admitted attempts per second.
IF global_tokens<1 THEN RETURN ceil((1-global_tokens)/2.0)::integer; END IF;
INSERT INTO login_buckets VALUES ('account:'||account,8,moment) ON CONFLICT DO NOTHING;
SELECT least(8.0,tokens+extract(epoch FROM moment-updated_at)*(8.0/60.0))
INTO account_tokens FROM login_buckets WHERE bucket='account:'||account FOR UPDATE;
IF account_tokens<1 THEN RETURN ceil((1-account_tokens)/(8.0/60.0))::integer; END IF;
UPDATE login_buckets SET tokens=global_tokens-1,updated_at=moment WHERE bucket='global';
UPDATE login_buckets SET tokens=account_tokens-1,updated_at=moment WHERE bucket='account:'||account;
DELETE FROM login_buckets WHERE bucket IN (
SELECT bucket FROM login_buckets WHERE updated_at<moment-interval '10 minutes'
AND bucket<>'global' ORDER BY updated_at LIMIT 128
);
RETURN 0;
END $$;
REVOKE ALL ON FUNCTION login_wait(text) FROM PUBLIC;
+60
View File
@@ -0,0 +1,60 @@
[project]
name = "mtusi-messenger-lab7"
version = "1.0.0"
requires-python = ">=3.14,<3.15"
dependencies = [
"fastapi==0.143.3",
"pydantic==2.14.1",
"uvicorn==0.54.0",
"psycopg[binary,pool]==3.3.6",
"argon2-cffi>=25,<26",
"redis==8.1.0",
"minio==7.2.20",
"pillow==13.0.0",
"opentelemetry-api==1.45.2",
"opentelemetry-sdk==1.45.2",
"opentelemetry-exporter-otlp-proto-http==1.45.2",
"psutil==7.1.3",
"cryptography==50.0.2",
]
[dependency-groups]
dev = [
"pytest>=8,<10",
"httpx>=0.28,<1",
"ruff>=0.14,<1",
"openapi-spec-validator==0.7.2",
"psutil>=7,<8",
"matplotlib>=3.10,<4",
"pyyaml==6.0.3",
"pip-audit==2.10.1",
]
[tool.ruff]
line-length = 79
target-version = "py314"
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
[tool.ruff.lint.per-file-ignores]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
"src/observer.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
+128
View File
@@ -0,0 +1,128 @@
"""Summarize measured fault invariants and child resource samples."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def effects(case):
for key in ["effect", "v1", "v2"]:
if key in case:
yield key, case[key]
for i, item in enumerate(case.get("effects", [])):
yield str(i + 1), item
def main():
p = argparse.ArgumentParser()
p.add_argument("input", type=Path)
p.add_argument("--output", required=True, type=Path)
a = p.parse_args()
d = json.loads(a.input.read_text())
assert d["complete"] and d.get("scope", "all") == "all"
rows = []
for case in d["cases"]:
for role, e in effects(case):
j = e["job"]
rows.append(
dict(
case=case["case"],
role=role,
job_id=j["id"],
version=j["pipeline_version"],
attempt=j["attempt"],
cycle=j["cycle"],
fence=j["fence"],
state=j["state"],
publications=e["publications"],
variant_rows=len(e["variants"]),
physical_keys=len(e["physical_keys"]),
)
)
a.output.mkdir(parents=True, exist_ok=True)
with (a.output / "effects.csv").open("w") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
ready = [r for r in rows if r["state"] == "ready"]
assert all(
r["publications"] == 1
and r["variant_rows"] == 4
and r["physical_keys"] == 4
for r in ready
)
stale = [r for r in rows if r["state"] == "obsolete"]
assert all(r["publications"] == 0 for r in stale)
job_ids = {r["job_id"] for r in rows}
samples = [
x["detail"]
for x in d["worker_metrics"]
if str(x["job_id"]) in job_ids
and x["detail"].get("result") == "ready"
and "rss_peak_mib" in x["detail"]
]
stats = {
"source_commit": d["source_commit"],
"cases": len(d["cases"]),
"observed_ready_effects": len(ready),
"observed_obsolete_effects": len(stale),
"all_ready_have_one_publication_and_four_variants": True,
"all_obsolete_have_zero_publications": True,
"successful_child_samples": len(samples),
"child_elapsed_range_seconds": [
min(x["seconds"] for x in samples),
max(x["seconds"] for x in samples),
],
"child_cpu_range_seconds": [
min(x["cpu_seconds"] for x in samples),
max(x["cpu_seconds"] for x in samples),
],
"child_peak_rss_range_mib": [
min(x["rss_peak_mib"] for x in samples),
max(x["rss_peak_mib"] for x in samples),
],
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
}
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
axes[0].bar(
range(1, len(rows) + 1),
[r["publications"] for r in rows],
color=[
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
],
)
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
axes[0].set(
xlabel="Effect observation (CSV row)",
ylabel="Committed publications",
title="One publication per ready version; stale v1 = 0",
yticks=[0, 1],
)
axes[1].scatter(
[x["seconds"] for x in samples],
[x["rss_peak_mib"] for x in samples],
color="#246b8e",
s=35,
alpha=0.8,
)
axes[1].set(
xlabel="Child elapsed, seconds",
ylabel="Child peak RSS, MiB",
title=f"{len(samples)} completed child samples (synthetic320×200)",
)
for ax in axes:
ax.grid(axis="y", alpha=0.2)
fig.savefig(a.output / "pipeline-results.png", dpi=150)
plt.close(fig)
print(json.dumps(stats, indent=2))
if __name__ == "__main__":
main()
+249
View File
@@ -0,0 +1,249 @@
"""Derive tables and a figure from completed observations, never simulated data."""
import argparse
import csv
import json
import math
from collections import Counter
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def quantile(values, fraction):
ordered = sorted(values)
return (
ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
if ordered
else None
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("input", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
report = json.loads(args.input.read_text())
assert report["complete"] and report["scope"] == "full30m"
assert report["elapsed_seconds"] >= 1800
with (args.input.parent / report["requests_csv"]).open() as stream:
requests = list(csv.DictReader(stream))
samples = report["samples"]
origin = samples[0]["at"]
summaries = []
for phase in report["phases"]:
rows = [r for r in requests if r["phase"] == phase["name"]]
successful = [
float(r["seconds"]) for r in rows if r["status"] == "201"
]
elapsed = phase["end"] - phase["start"]
summaries.append(
{
"phase": phase["name"],
"seconds": round(elapsed, 3),
"started_requests": len(rows),
"statuses": dict(Counter(r["status"] for r in rows)),
"successful_rps": round(len(successful) / elapsed, 3),
"success_p50_ms": round(quantile(successful, 0.5) * 1000, 3)
if successful
else None,
"success_p95_ms": round(quantile(successful, 0.95) * 1000, 3)
if successful
else None,
"success_p99_ms": round(quantile(successful, 0.99) * 1000, 3)
if successful
else None,
"critical_firing_names": sorted(
n
for n in phase["firing"]
if n
in {
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
}
),
"warning_firing_names": sorted(
n
for n in phase["firing"]
if n
not in {
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
}
),
}
)
intervals = {}
for name in sorted({a["name"] for s in samples for a in s["alerts"]}):
segments, start = [], None
for sample in samples:
active = any(
a["name"] == name and a["state"] == "firing"
for a in sample["alerts"]
)
if active and start is None:
start = sample["at"]
elif not active and start is not None:
segments.append([start, sample["at"]])
start = None
if start is not None:
segments.append([start, samples[-1]["at"]])
intervals[name] = segments
gap = [
b["at"] - a["at"] for a, b in zip(samples, samples[1:], strict=False)
]
memory = [
s["signals"]["collector_memory"] / 1024**2
for s in samples
if s["signals"]["collector_memory"] is not None
]
summary = {
"source_commit": report["source_commit"],
"runtime_versions": report["runtime_versions"],
"elapsed_seconds": report["elapsed_seconds"],
"sample_count": len(samples),
"sample_gap_seconds": {
"min": min(gap),
"median": quantile(gap, 0.5),
"max": max(gap),
},
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
"phases": summaries,
"incidents": report["incidents"],
"firing_intervals_unix_seconds": intervals,
"short_spikes": report["short_spikes"],
"collector_rss_mib": {"min": min(memory), "max": max(memory)},
"collector_max_queued_packets_sum": max(
s["signals"]["collector_queue"] or 0 for s in samples
),
"collector_queue_capacity_sum": max(
s["signals"]["collector_queue_capacity"] or 0 for s in samples
),
"backend_outage": report["backend_outage"],
"preservation": report["preservation"],
"final_30m_sli": report["final_30m_sli"],
}
args.output.mkdir(parents=True, exist_ok=True)
(args.output / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n"
)
with (args.output / "phases.csv").open("w") as stream:
writer = csv.DictWriter(stream, fieldnames=list(summaries[0]))
writer.writeheader()
writer.writerows(summaries)
x = [(s["at"] - origin) / 60 for s in samples]
fig, axes = plt.subplots(
4, 1, figsize=(12, 12), sharex=True, layout="constrained"
)
for axis in axes:
axis.grid(alpha=0.2)
for phase in report["phases"]:
if (
phase["name"].endswith("_sustained")
or phase["name"] == "backend_unavailable"
):
color = (
"#cf704e"
if phase["name"].startswith("api")
else "#8b79b5"
if phase["name"].startswith("ready")
else "#85919d"
)
axis.axvspan(
(phase["start"] - origin) / 60,
(phase["end"] - origin) / 60,
color=color,
alpha=0.15,
)
axes[0].plot(
x,
[(s["signals"]["api_error_ratio_1m"] or 0) * 100 for s in samples],
color="#b04a32",
label="API 5xx ratio, rolling 1 min",
)
axes[0].axhline(1, color="black", ls=":", label="1% error budget")
axes[0].set_ylabel("5xx, %")
axes[0].legend(loc="upper right")
axes[0].set_title(
"Actual 30-minute SLO experiment: DB delay, stopped worker, unavailable telemetry stores"
)
axes[1].plot(
x,
[s["signals"]["queue_depth"] for s in samples],
color="#386a9b",
label="Queued jobs",
)
axes[1].set_ylabel("Jobs")
other = axes[1].twinx()
other.plot(
x,
[s["signals"]["oldest_job_age"] for s in samples],
color="#8b579d",
label="Oldest unfinished job",
)
other.axhline(20, ls=":", color="#8b579d")
other.set_ylabel("Age, seconds")
axes[1].legend(loc="upper left")
other.legend(loc="upper right")
alert_names = [
"ApiFastBurn",
"ApiSlowBurn",
"ReadyFastBurn",
"ReadySlowBurn",
]
for index, name in enumerate(alert_names):
axes[2].broken_barh(
[
((start - origin) / 60, (end - start) / 60)
for start, end in intervals.get(name, [])
],
(index - 0.3, 0.6),
facecolors="#bd523c",
)
axes[2].set_yticks(range(4), alert_names)
axes[2].set_ylim(-0.6, 3.6)
axes[2].set_ylabel("Critical alert firing")
axes[3].plot(
x,
[
s["signals"]["collector_memory"] / 1024**2
if s["signals"]["collector_memory"] is not None
else None
for s in samples
],
color="#347465",
label="Collector RSS",
)
axes[3].axhline(
report["container_limits"]["otel"]["mem_limit"] / 1024**2,
ls=":",
color="black",
label="Container memory limit",
)
axes[3].set_ylabel("MiB")
axes[3].set_xlabel("Minutes from first observation")
other = axes[3].twinx()
other.plot(
x,
[s["signals"]["collector_queue"] for s in samples],
color="#9c6e2e",
label="Queued telemetry packets, sum",
)
other.set_ylabel("Packets")
axes[3].legend(loc="upper left")
other.legend(loc="upper right")
fig.savefig(args.output / "slo-timeline.png", dpi=150)
plt.close(fig)
print("PASS: measured phase statistics, alert intervals and timeline")
if __name__ == "__main__":
main()
+351
View File
@@ -0,0 +1,351 @@
"""Explicit operator init/unseal/configure; no unseal material enters containers."""
import argparse
import json
import os
import ssl
import time
import httpx
from cryptography import x509
from scripts.bootstrap_security import (
OPERATOR,
ROLES,
ROOT,
TRUST,
ca,
public,
sign_intermediate,
write_private,
)
class Operator:
def __init__(self):
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
context.minimum_version = ssl.TLSVersion.TLSv1_3
context.load_cert_chain(
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
)
self.http = httpx.Client(
base_url="https://localhost:"
+ os.getenv("BAO_PORT", "8207")
+ "/v1/",
verify=context,
trust_env=False,
timeout=10,
)
self.token = None
def close(self):
self.http.close()
def call(self, method, path, payload=None, *, token=None):
response = self.http.request(
method,
path,
json=payload,
headers={"X-Vault-Token": token} if token else {},
)
if response.status_code >= 300:
raise RuntimeError(
"Operator request denied/unavailable: "
+ path
+ " status "
+ str(response.status_code)
)
return response.json() if response.content else {}
def init(self):
if self.call("GET", "sys/init")["initialized"]:
return
result = self.call(
"POST", "sys/init", {"secret_shares": 3, "secret_threshold": 2}
)
for index, key in enumerate(result["keys_base64"]):
write_private(
OPERATOR / ("unseal-" + str(index + 1) + ".json"),
json.dumps({"key": key}) + "\n",
)
write_private(OPERATOR / "bootstrap-root-token", result["root_token"])
def unseal(self):
state = self.call("GET", "sys/seal-status")
if not state["sealed"]:
return
for index in [1, 2]:
share = json.loads(
(OPERATOR / ("unseal-" + str(index) + ".json")).read_text()
)
state = self.call("POST", "sys/unseal", {"key": share["key"]})
if not state["sealed"]:
return
raise RuntimeError("Unseal threshold not reached")
def authenticate(self):
cached = OPERATOR / "operator-token.json"
if cached.exists():
data = json.loads(cached.read_text())
if data["expires_at"] > time.time() + 30:
self.token = data["token"]
return self.token
result = self.call("POST", "auth/cert/login", {})["auth"]
self.token = result["client_token"]
write_private(
cached,
json.dumps(
{
"token": self.token,
"expires_at": time.time() + result["lease_duration"],
}
)
+ "\n",
)
return self.token
def authorized(self, method, path, payload=None):
return self.call(
method, path, payload, token=self.token or self.authenticate()
)
def configure(self):
marker = OPERATOR / "configured.json"
if marker.exists():
self.authenticate()
self.authorized("GET", "auth/token/lookup-self")
return
token = (OPERATOR / "bootstrap-root-token").read_text().strip()
def root(method, path, payload=None):
return self.call(method, path, payload, token=token)
mounts = root("GET", "sys/mounts")
mounts = mounts.get("data", mounts)
if "secret/" not in mounts:
root(
"POST",
"sys/mounts/secret",
{"type": "kv", "options": {"version": "2"}},
)
if "pki/" not in mounts:
root(
"POST",
"sys/mounts/pki",
{"type": "pki", "config": {"max_lease_ttl": "720h"}},
)
auth = root("GET", "sys/auth")
auth = auth.get("data", auth)
if "cert/" not in auth:
root("POST", "sys/auth/cert", {"type": "cert"})
issuer_marker = OPERATOR / "initial-issuer.json"
if not issuer_marker.exists():
csr = root(
"POST",
"pki/intermediate/generate/internal",
{
"common_name": "Messenger service intermediate",
"key_type": "ec",
"key_bits": 256,
},
)["data"]["csr"]
key, cert = ca()
intermediate = sign_intermediate(
key, cert, x509.load_pem_x509_csr(csr.encode())
)
imported = root(
"POST",
"pki/intermediate/set-signed",
{
"certificate": (
public(intermediate) + public(cert)
).decode()
},
)
issuers = imported["data"]["imported_issuers"]
selected = None
for issuer in issuers:
details = root("GET", "pki/issuer/" + issuer)["data"]
loaded = x509.load_pem_x509_certificate(
details["certificate"].encode()
)
if loaded.serial_number == intermediate.serial_number:
selected = issuer
assert selected, "Imported signing intermediate not found"
root(
"POST",
"pki/config/issuers",
{"default": selected, "default_follows_latest_issuer": False},
)
write_private(
issuer_marker,
json.dumps(
{
"configured": True,
"imported_issuers": imported.get("data", {}).get(
"imported_issuers", []
),
}
)
+ "\n",
)
_, cert = ca()
material = json.loads((OPERATOR / "service-secrets.json").read_text())
for role, cfg in ROLES.items():
policy = (
ROOT / "infra/openbao/policies" / (role + ".hcl")
).read_text()
if cfg["kv"]:
root(
"POST",
"secret/data/" + role,
{"data": material["services"][role]["values"]},
)
root(
"PUT", "sys/policies/acl/messenger-" + role, {"policy": policy}
)
root(
"POST",
"pki/roles/" + role,
{
"allowed_domains": cfg["dns"],
"allow_bare_domains": True,
"allow_subdomains": False,
"allow_wildcard_certificates": False,
"allow_any_name": False,
"allow_localhost": "localhost" in cfg["dns"],
"allow_ip_sans": bool(cfg.get("ip")),
"allowed_uri_sans": ["spiffe://messenger/" + role],
"enforce_hostnames": True,
"key_type": "ec",
"key_bits": 256,
"server_flag": True,
"client_flag": True,
"ttl": "120s",
"max_ttl": "120s",
"issuer_ref": "default",
},
)
root(
"POST",
"auth/cert/certs/" + role + "-v1",
{
"display_name": role,
"certificate": public(cert).decode(),
"allowed_common_names": [role],
"allowed_uri_sans": ["spiffe://messenger/" + role],
"token_policies": ["messenger-" + role],
"token_ttl": "120s",
"token_max_ttl": "600s",
"token_no_default_policy": True,
},
)
operator_policy = (
ROOT / "infra/openbao/policies/operator.hcl"
).read_text()
root(
"PUT",
"sys/policies/acl/messenger-operator",
{"policy": operator_policy},
)
root(
"POST",
"auth/cert/certs/operator-v1",
{
"display_name": "operator",
"certificate": public(cert).decode(),
"allowed_common_names": ["operator"],
"allowed_uri_sans": ["spiffe://messenger/operator"],
"token_policies": ["messenger-operator"],
"token_ttl": "900s",
"token_max_ttl": "3600s",
"token_no_default_policy": True,
},
)
audit = root("GET", "sys/audit")
audit = audit.get("data", audit)
if "file/" not in audit:
root(
"PUT",
"sys/audit/file",
{
"type": "file",
"options": {
"file_path": "/data/audit/audit.log",
"mode": "0600",
},
},
)
created = root(
"POST",
"auth/token/create-orphan",
{
"policies": ["messenger-operator"],
"ttl": "900s",
"renewable": True,
"no_default_policy": True,
},
)["auth"]
write_private(
OPERATOR / "operator-token.json",
json.dumps(
{
"token": created["client_token"],
"expires_at": time.time() + created["lease_duration"],
}
)
+ "\n",
)
root("POST", "auth/token/revoke-self", {})
write_private(
marker,
json.dumps({"configured": True, "root_token_revoked": True})
+ "\n",
)
(OPERATOR / "bootstrap-root-token").unlink()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"action", choices=["init", "unseal", "configure", "status", "seal"]
)
args = parser.parse_args()
operator = Operator()
try:
if args.action == "init":
operator.init()
operator.unseal()
operator.configure()
elif args.action == "unseal":
operator.unseal()
elif args.action == "configure":
operator.configure()
elif args.action == "seal":
operator.authorized("POST", "sys/seal", {})
else:
state = operator.call("GET", "sys/seal-status")
print(
json.dumps(
{
k: state[k]
for k in [
"initialized",
"sealed",
"version",
"storage_type",
]
if k in state
}
)
)
if args.action != "status":
print(
"Operator action completed; no tokens or unseal shares printed"
)
finally:
operator.close()
if __name__ == "__main__":
main()
+64
View File
@@ -0,0 +1,64 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "grafana_password", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+436
View File
@@ -0,0 +1,436 @@
"""Operator-only offline trust/bootstrap generation; never prints key material."""
import argparse
import hashlib
import ipaddress
import json
import os
import secrets
from datetime import UTC, datetime, timedelta
from pathlib import Path
from urllib.parse import quote
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
ROOT = Path(__file__).resolve().parents[1]
OPERATOR = ROOT / "operator"
SECRET = ROOT / "secrets"
TRUST = ROOT / "trust"
ROLES = {
"openbao": {
"dns": ["openbao", "localhost"],
"ip": ["127.0.0.1"],
"uid": 1000,
"kv": False,
},
"api": {"dns": ["api", "localhost"], "uid": 10001, "kv": True},
"worker": {"dns": ["worker"], "uid": 10001, "kv": True},
"relay": {"dns": ["relay"], "uid": 10001, "kv": True},
"observer": {"dns": ["observer"], "uid": 10001, "kv": True},
"reconciler": {"dns": ["reconciler"], "uid": 10001, "kv": True},
"migrate": {"dns": ["migrate"], "uid": 10001, "kv": True},
"lb": {
"dns": ["lb", "localhost"],
"ip": ["127.0.0.1"],
"uid": 99,
"kv": False,
},
"postgres": {"dns": ["postgres"], "uid": 10001, "kv": False},
"s3": {
"dns": ["s3", "localhost"],
"ip": ["127.0.0.1"],
"uid": 10001,
"kv": False,
},
"broker": {"dns": ["broker"], "uid": 10001, "kv": False},
"cache": {"dns": ["cache"], "uid": 10001, "kv": False},
"otel": {"dns": ["otel"], "uid": 10001, "kv": False},
"prometheus": {"dns": ["prometheus"], "uid": 65534, "kv": False},
"loki": {"dns": ["loki"], "uid": 10001, "kv": False},
"tempo": {"dns": ["tempo"], "uid": 10001, "kv": False},
"grafana": {
"dns": ["grafana", "localhost"],
"ip": ["127.0.0.1"],
"uid": 472,
"kv": False,
},
}
def write_private(path, content, *, mode=0o600):
path.parent.mkdir(parents=True, exist_ok=True)
raw = content.encode() if isinstance(content, str) else content
temporary = path.with_name(path.name + ".new")
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
try:
os.fchmod(fd, mode)
with os.fdopen(fd, "wb") as stream:
stream.write(raw)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
temporary.unlink(missing_ok=True)
raise
temporary.replace(path)
def key_bytes(key):
return key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
def public(cert):
return cert.public_bytes(serialization.Encoding.PEM)
def make_root():
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name(
[
x509.NameAttribute(
NameOID.COMMON_NAME,
"Messenger lab root " + secrets.token_hex(4),
)
]
)
now = datetime.now(UTC)
cert = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - timedelta(minutes=1))
.not_valid_after(now + timedelta(days=365))
.add_extension(
x509.BasicConstraints(ca=True, path_length=1), critical=True
)
.add_extension(
x509.KeyUsage(
digital_signature=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=True,
crl_sign=True,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
.add_extension(
x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
critical=False,
)
.sign(key, hashes.SHA256())
)
return key, cert
def sign_leaf(ca_key, ca_cert, role, dns, ips=(), *, seconds=1800):
key = ec.generate_private_key(ec.SECP256R1())
now = datetime.now(UTC)
names = [x509.DNSName(name) for name in dns]
names += [x509.IPAddress(ipaddress.ip_address(ip)) for ip in ips]
names += [x509.UniformResourceIdentifier("spiffe://messenger/" + role)]
cert = (
x509.CertificateBuilder()
.subject_name(
x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, role)])
)
.issuer_name(ca_cert.subject)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - timedelta(seconds=30))
.not_valid_after(now + timedelta(seconds=seconds))
.add_extension(
x509.BasicConstraints(ca=False, path_length=None), critical=True
)
.add_extension(x509.SubjectAlternativeName(names), critical=False)
.add_extension(
x509.ExtendedKeyUsage(
[
ExtendedKeyUsageOID.CLIENT_AUTH,
ExtendedKeyUsageOID.SERVER_AUTH,
]
),
critical=False,
)
.add_extension(
x509.KeyUsage(
digital_signature=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=False,
crl_sign=False,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
.add_extension(
x509.AuthorityKeyIdentifier.from_issuer_public_key(
ca_key.public_key()
),
critical=False,
)
.sign(ca_key, hashes.SHA256())
)
return key, cert
def sign_intermediate(ca_key, ca_cert, csr):
now = datetime.now(UTC)
assert csr.is_signature_valid
return (
x509.CertificateBuilder()
.subject_name(csr.subject)
.issuer_name(ca_cert.subject)
.public_key(csr.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - timedelta(seconds=30))
.not_valid_after(now + timedelta(days=30))
.add_extension(
x509.BasicConstraints(ca=True, path_length=0), critical=True
)
.add_extension(
x509.KeyUsage(
digital_signature=True,
content_commitment=False,
key_encipherment=False,
data_encipherment=False,
key_agreement=False,
key_cert_sign=True,
crl_sign=True,
encipher_only=False,
decipher_only=False,
),
critical=True,
)
.add_extension(
x509.SubjectKeyIdentifier.from_public_key(csr.public_key()),
critical=False,
)
.add_extension(
x509.AuthorityKeyIdentifier.from_issuer_public_key(
ca_key.public_key()
),
critical=False,
)
.sign(ca_key, hashes.SHA256())
)
def ca():
return (
serialization.load_pem_private_key(
(OPERATOR / "ca-key.pem").read_bytes(), password=None
),
x509.load_pem_x509_certificate((OPERATOR / "ca.pem").read_bytes()),
)
def generate():
for directory in [OPERATOR, SECRET]:
directory.mkdir(mode=0o700, exist_ok=True)
directory.chmod(0o700)
TRUST.mkdir(mode=0o755, exist_ok=True)
if not (OPERATOR / "ca-key.pem").exists():
key, cert = make_root()
write_private(OPERATOR / "ca-key.pem", key_bytes(key))
write_private(OPERATOR / "ca.pem", public(cert))
key, cert = ca()
if not (TRUST / "ca.pem").exists():
write_private(TRUST / "ca.pem", public(cert), mode=0o644)
write_private(TRUST / "revoked.json", "{}\n", mode=0o644)
for role, cfg in {
**ROLES,
"openbao": {"dns": ["openbao", "localhost"], "ip": ["127.0.0.1"]},
"operator": {"dns": ["operator"]},
}.items():
directory = (
(OPERATOR / "client")
if role == "operator"
else SECRET / "bootstrap" / role
)
renew_operator = (
role == "operator"
and (directory / "cert.pem").exists()
and x509.load_pem_x509_certificate(
(directory / "cert.pem").read_bytes()
).not_valid_after_utc
< datetime.now(UTC) + timedelta(hours=1)
)
renew_bootstrap = (
role != "operator"
and (directory / "cert.pem").exists()
and x509.load_pem_x509_certificate(
(directory / "cert.pem").read_bytes()
).not_valid_after_utc
< datetime.now(UTC) + timedelta(minutes=5)
)
wrong_issuer = (
directory / "cert.pem"
).exists() and x509.load_pem_x509_certificate(
(directory / "cert.pem").read_bytes()
).issuer != cert.subject
# Only this explicit operator command has the offline CA. Agents
# cannot refresh a bootstrap credential by themselves after expiry.
if (
not (directory / "key.pem").exists()
or renew_operator
or renew_bootstrap
or wrong_issuer
):
leaf_key, leaf = sign_leaf(
key,
cert,
role,
cfg["dns"],
cfg.get("ip", ()),
seconds=86400 if role in {"operator", "openbao"} else 1800,
)
directory.mkdir(parents=True, exist_ok=True)
directory.chmod(0o700 if role == "operator" else 0o755)
mode = 0o600 if role == "operator" else 0o644
write_private(
directory / "key.pem", key_bytes(leaf_key), mode=mode
)
write_private(
directory / "cert.pem", public(leaf) + public(cert), mode=mode
)
write_private(
directory / "bundle.pem",
public(leaf) + public(cert) + key_bytes(leaf_key),
mode=mode,
)
path = OPERATOR / "service-secrets.json"
if path.exists():
raw = json.loads(path.read_text())
else:
raw = {
name: secrets.token_urlsafe(32)
for name in [
"postgres_password",
"s3_root_password",
"broker_admin",
"cache_admin",
"cache_api",
"grafana_password",
]
}
raw["s3_root_user"] = "operator-" + secrets.token_hex(12)
raw["services"] = {}
for role in [
"api",
"worker",
"relay",
"observer",
"reconciler",
"migrate",
]:
pwd = secrets.token_urlsafe(32)
values = {
"DATABASE_URL": "postgresql://"
+ ("messenger" if role == "migrate" else "svc_" + role)
+ ":"
+ quote(
raw["postgres_password"] if role == "migrate" else pwd,
safe="",
)
+ "@postgres:5443/messenger?sslmode=verify-full&gssencmode=disable&sslrootcert=/trust/ca.pem&sslcert=/run/identity/current/cert.pem&sslkey=/run/identity/current/key.pem"
}
service = {"database_password": pwd, "values": values}
if role == "api":
values["CACHE_URL"] = (
"rediss://api:" + raw["cache_api"] + "@cache:6380/0"
)
if role in {"relay", "worker"}:
service["broker_password"] = secrets.token_urlsafe(32)
values["BROKER_URL"] = (
"rediss://"
+ role
+ ":"
+ service["broker_password"]
+ "@broker:6380/0"
)
if role in {"api", "worker", "reconciler"}:
values.update(
S3_ACCESS_KEY=role + "-" + secrets.token_hex(12),
S3_SECRET_KEY=secrets.token_urlsafe(32),
)
raw["services"][role] = service
write_private(path, json.dumps(raw, indent=2) + "\n")
for name in [
"postgres_password",
"s3_root_user",
"s3_root_password",
"grafana_password",
]:
write_private(SECRET / name, raw[name] + "\n", mode=0o644)
for name in ["broker", "cache"]:
admin = raw[name + "_admin"]
lines = [
"bind 127.0.0.1",
"port 6379",
"protected-mode yes",
'save ""',
"maxmemory 64mb" if name == "cache" else "maxmemory 192mb",
"maxmemory-policy allkeys-lru"
if name == "cache"
else "maxmemory-policy noeviction",
"user default off",
"user admin on #"
+ hashlib.sha256(admin.encode()).hexdigest()
+ " ~* &* +@all",
]
if name == "cache":
lines += [
"user api on #"
+ hashlib.sha256(raw["cache_api"].encode()).hexdigest()
+ " ~user:* +get +set +ping +select +client|setinfo +client|setname"
]
else:
lines += ["appendonly yes", "appendfsync always", "dir /data"]
for role, commands in [
("relay", "+xadd +waitaof"),
(
"worker",
"+xgroup|create +xreadgroup +xautoclaim +xack +xdel",
),
]:
lines += [
"user "
+ role
+ " on #"
+ hashlib.sha256(
raw["services"][role]["broker_password"].encode()
).hexdigest()
+ " ~attachments +ping +select +client|setinfo +client|setname "
+ commands
]
write_private(
SECRET / (name + ".conf"), "\n".join(lines) + "\n", mode=0o644
)
write_private(SECRET / (name + "_admin"), admin + "\n", mode=0o644)
def main():
argparse.ArgumentParser(description=__doc__).parse_args()
generate()
print(
"Local operator trust, scoped service credentials and bootstrap certificates prepared; existing secrets retained"
)
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+231
View File
@@ -0,0 +1,231 @@
"""Operator helpers for explicit certificate repair and working-key rotation."""
import argparse
import io
import json
import os
import secrets
import ssl
import time
from uuid import uuid4
import urllib3
from minio import Minio
from minio.credentials import StaticProvider
from minio.error import MinioAdminException, S3Error
from minio.minioadmin import MinioAdmin
from scripts.bao_operator import Operator
from scripts.bootstrap_security import OPERATOR, ROLES, TRUST, write_private
from scripts.prepare_dependencies import attach_policy
from scripts.stand import compose
INSTALL = """import json,os,pathlib,signal,sys,tempfile
data=json.load(sys.stdin)
base=pathlib.Path('/run/identity')
directory=pathlib.Path(tempfile.mkdtemp(prefix='generation-',dir=base))
certificate=data['certificate']+'\\n'+'\\n'.join(data.get('ca_chain',[]))
for name,value in [('cert.pem',certificate),('key.pem',data['private_key']),('bundle.pem',certificate+'\\n'+data['private_key'])]:
path=directory/name;path.write_text(value);path.chmod(0o600)
link=base/'operator-next';link.unlink(missing_ok=True);link.symlink_to(directory.name);link.replace(base/'current')
if data.get('reload_lb'):os.kill(1,signal.SIGUSR2)
print('Operator installed a scoped short-lived leaf')
"""
def install(role, data):
payload = {**data, "reload_lb": role == "lb"}
args = ["--profile", "maintenance"] if role == "migrate" else []
compose(
*args,
"run",
"--rm",
"--no-deps",
"-T",
"--entrypoint",
"python",
"identity-" + role,
"-c",
INSTALL,
input=json.dumps(payload),
)
def issue(operator, role):
cfg = ROLES[role]
request = {
"common_name": role,
"alt_names": ",".join(cfg["dns"]),
"uri_sans": "spiffe://messenger/" + role,
"ttl": "120s",
}
if cfg.get("ip"):
request["ip_sans"] = ",".join(cfg["ip"])
return operator.authorized("POST", "pki/issue/" + role, request)["data"]
def repair(operator, role):
result = issue(operator, role)
install(role, result)
return {key: result[key] for key in ["serial_number", "expiration"]}
def admin():
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
context.minimum_version = ssl.TLSVersion.TLSv1_3
context.load_cert_chain(
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
)
pool = urllib3.PoolManager(
ssl_context=context,
timeout=urllib3.Timeout(connect=2, read=8),
retries=False,
)
return (
raw,
MinioAdmin(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=StaticProvider(
raw["s3_root_user"], raw["s3_root_password"]
),
secure=True,
http_client=pool,
),
pool,
)
def rotate_s3(operator):
raw, client, pool = admin()
try:
journal_path = OPERATOR / "s3-rotation.json"
resumed = journal_path.exists() and not json.loads(
journal_path.read_text()
).get("complete")
journal = json.loads(journal_path.read_text()) if resumed else {}
values = journal.get(
"previous_values", raw["services"]["api"]["values"]
)
previous = values["S3_ACCESS_KEY"]
old = Minio(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=StaticProvider(previous, values["S3_SECRET_KEY"]),
secure=True,
http_client=pool,
region="us-east-1",
)
probe_key = journal.get(
"probe_key", "objects/staging/rotation-" + uuid4().hex
)
current = journal.get("next_values") or {
**values,
"S3_ACCESS_KEY": "api-" + secrets.token_hex(12),
"S3_SECRET_KEY": secrets.token_urlsafe(32),
}
if not resumed:
old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
journal = {
"previous_values": values,
"next_values": current,
"probe_key": probe_key,
"complete": False,
}
write_private(journal_path, json.dumps(journal) + "\n")
client.user_add(current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"])
attach_policy(client, "messenger-api", current["S3_ACCESS_KEY"])
operator.authorized("POST", "secret/data/api", {"data": current})
# Compare a fingerprint; credential values are not placed in argv/output.
import hashlib
expected = hashlib.sha256(
current["S3_ACCESS_KEY"].encode()
).hexdigest()
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
observed = compose(
"exec",
"-T",
"identity-api",
"python",
"-c",
"import hashlib,json;print(hashlib.sha256(json.load(open('/run/identity/secrets.json'))['values']['S3_ACCESS_KEY'].encode()).hexdigest())",
)
if observed == expected:
break
time.sleep(1)
else:
raise TimeoutError("New scoped S3 key not observed")
raw["services"]["api"]["values"] = current
write_private(
OPERATOR / "service-secrets.json", json.dumps(raw, indent=2) + "\n"
)
try:
client.user_remove(previous)
except MinioAdminException as error:
if json.loads(error._body).get("Code") != "XMinioAdminNoSuchUser":
raise
pool.clear()
try:
old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
except S3Error as error:
assert error.code in {"AccessDenied", "InvalidAccessKeyId"}, (
error.code
)
else:
raise AssertionError(
"Removed working key accepted on a fresh connection"
)
fresh = Minio(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=StaticProvider(
current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"]
),
secure=True,
http_client=pool,
region="us-east-1",
)
fresh.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
root = Minio(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=StaticProvider(
raw["s3_root_user"], raw["s3_root_password"]
),
secure=True,
http_client=pool,
region="us-east-1",
)
root.remove_object("messenger", probe_key)
write_private(
journal_path, json.dumps({**journal, "complete": True}) + "\n"
)
return {
"new_key_observed": True,
"old_account_removed": True,
"old_key_fresh_connection_denied": True,
"new_key_put_succeeded": True,
"image_rebuilt": False,
}
finally:
pool.clear()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["repair-leaf", "rotate-s3"])
parser.add_argument("--role", choices=ROLES, default="lb")
args = parser.parse_args()
operator = Operator()
try:
result = (
repair(operator, args.role)
if args.action == "repair-leaf"
else rotate_s3(operator)
)
print(json.dumps(result))
finally:
operator.close()
if __name__ == "__main__":
main()
+154
View File
@@ -0,0 +1,154 @@
"""Read-only queries through an internal probe, without exposing admin ports."""
import base64
import json
import time
from urllib.parse import urlencode
from scripts.stand import compose
ENDPOINTS = {
"prometheus": "https://prometheus:9900",
"loki": "https://loki:3310",
"tempo": "https://tempo:3320",
"otel": "https://otel:13333",
}
def fetch(service, path, *, params=None, json_result=True):
url = ENDPOINTS[service] + path
if params:
url += "?" + urlencode(params, doseq=True)
code = (
f"import urllib.request; from src.security_probe import context; service={service!r}; "
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); "
"print(r.read().decode())"
)
text = compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
code,
)
return json.loads(text) if json_result else text
def query(expression, *, at=None):
params = {"query": expression}
if at is not None:
params["time"] = str(at)
result = fetch("prometheus", "/api/v1/query", params=params)
assert result["status"] == "success"
return result["data"]["result"]
def scalar(expression):
result = query(expression)
return float(result[0]["value"][1]) if result else 0.0
def queries(expressions):
urls = {
name: ENDPOINTS["prometheus"]
+ "/api/v1/query?"
+ urlencode({"query": expression})
for name, expression in expressions.items()
}
code = (
"import json,urllib.request\nfrom src.security_probe import context\n"
f"urls={urls!r}\n"
"result={}\n"
"for name,url in urls.items():\n"
" with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n"
"print(json.dumps(result))"
)
return json.loads(
compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
code,
)
)
def logs(trace_id, *, start=None):
return fetch(
"loki",
"/loki/api/v1/query_range",
params={
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
+ trace_id
+ '"',
"start": str(int((start or time.time() - 3600) * 1e9)),
"end": str(time.time_ns()),
"limit": 500,
},
)
def normalize_id(value):
if not value:
return ""
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
return value
return base64.b64decode(value).hex()
def spans(trace_id):
result = fetch("tempo", "/api/traces/" + trace_id)
flattened = []
for batch in result.get("batches", result.get("resourceSpans", [])):
resource = {
a["key"]: next(iter(a["value"].values()))
for a in batch.get("resource", {}).get("attributes", [])
}
for scope in batch.get("scopeSpans", []):
for item in scope.get("spans", []):
flattened.append(
{
"service": resource.get("service.name"),
"version": resource.get("service.version"),
"instance": resource.get("service.instance.id"),
"name": item["name"],
"kind": item.get("kind"),
"trace_id": normalize_id(item.get("traceId")),
"span_id": normalize_id(item.get("spanId")),
"parent_span_id": normalize_id(
item.get("parentSpanId")
),
"duration_seconds": (
int(item["endTimeUnixNano"])
- int(item["startTimeUnixNano"])
)
/ 1e9,
"status": item.get("status", {}),
"attributes": {
a["key"]: next(iter(a["value"].values()))
for a in item.get("attributes", [])
},
}
)
return flattened
def wait(condition, *, timeout=90, interval=1):
end = time.monotonic() + timeout
last_error = None
while time.monotonic() < end:
try:
result = condition()
if result:
return result
except (RuntimeError, ValueError, KeyError) as exc:
last_error = type(exc).__name__
time.sleep(interval)
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
+57
View File
@@ -0,0 +1,57 @@
"""Prepare a 60-second operator-only snapshot and run the audited pipeline CLI."""
import json
import sys
import time
from urllib.parse import quote
from scripts.bootstrap_security import OPERATOR, SECRET, write_private
from scripts.stand import compose
def prepare():
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
directory = SECRET / "operator-task"
directory.mkdir(mode=0o755, exist_ok=True)
for name in ["cert.pem", "key.pem"]:
write_private(
directory / "current" / name,
(OPERATOR / "client" / name).read_bytes(),
mode=0o644,
)
values = {
"DATABASE_URL": "postgresql://messenger:"
+ quote(raw["postgres_password"], safe="")
+ "@postgres:5443/messenger?sslmode=verify-full&gssencmode=disable&sslrootcert=/trust/ca.pem&sslcert=/run/identity/current/cert.pem&sslkey=/run/identity/current/key.pem",
"BROKER_URL": "rediss://admin:"
+ raw["broker_admin"]
+ "@broker:6380/0",
"S3_ACCESS_KEY": raw["s3_root_user"],
"S3_SECRET_KEY": raw["s3_root_password"],
}
write_private(
directory / "secrets.json",
json.dumps({"valid_until": time.time() + 60, "values": values}) + "\n",
mode=0o644,
)
def run(*args):
prepare()
return compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-task",
*args,
)
def main():
print(run(*sys.argv[1:]))
if __name__ == "__main__":
main()
+359
View File
@@ -0,0 +1,359 @@
"""Explicit operator PKI lifecycle; trust is never disabled."""
import argparse
import json
import time
from uuid import uuid4
from cryptography import x509
from cryptography.hazmat.primitives import hashes
from scripts.bao_operator import Operator
from scripts.bootstrap_security import (
OPERATOR,
ROLES,
SECRET,
TRUST,
ca,
key_bytes,
make_root,
public,
sign_intermediate,
sign_leaf,
write_private,
)
from scripts.credentials import install, repair
from scripts.stand import compose
REQUEST = (
b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n"
)
def fixture_check(role, *, reject=False):
expression = f"exchange('api',8443,{role!r},{REQUEST!r})"
assertion = (
"denied(lambda:" + expression + ")"
if reject
else "b' 200 ' in " + expression
)
code = (
"from src.security_probe import exchange,denied;assert "
+ assertion
+ ";print('Fresh TLS fixture check passed')"
)
compile(code, "fixture-check", "exec")
return compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
code,
)
META = """import json,pathlib
from cryptography import x509
result={}
for directory in pathlib.Path('/identities').iterdir():
try:
cert=x509.load_pem_x509_certificate((directory/'current/cert.pem').read_bytes())
result[directory.name]={'serial':format(cert.serial_number,'x'),'issuer':cert.issuer.rfc4514_string(),'expires_at':cert.not_valid_after_utc.timestamp()}
except FileNotFoundError:pass
print(json.dumps(result))
"""
def metadata():
return json.loads(
compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
META,
)
)
def reload_lb():
compose(
"exec",
"-T",
"identity-lb",
"python",
"-c",
"import os,signal;os.kill(1,signal.SIGUSR2)",
)
def preserve_lb():
data = json.loads(
compose(
"exec",
"-T",
"identity-lb",
"python",
"-c",
"import json,pathlib;p=pathlib.Path('/run/identity/current');print(json.dumps({name:(p/name).read_text() for name in ['cert.pem','key.pem']}))",
)
)
directory = SECRET / "probe/revoked-lb"
for name, value in data.items():
write_private(directory / name, value, mode=0o644)
leaf = x509.load_pem_x509_certificate(data["cert.pem"].encode())
return {
"serial": format(leaf.serial_number, "x"),
"expires_at": leaf.not_valid_after_utc.timestamp(),
}
def revoke(operator, leaf):
assert leaf["expires_at"] > time.time() + 5, (
"Revocation must be tested before natural expiry"
)
serial = leaf["serial"]
serial = ("0" + serial) if len(serial) % 2 else serial
colon = ":".join(serial[i : i + 2] for i in range(0, len(serial), 2))
operator.authorized("POST", "pki/revoke", {"serial_number": colon})
response = operator.http.get(
"pki/crl/pem",
headers={"X-Vault-Token": operator.token or operator.authenticate()},
)
response.raise_for_status()
operator.authorized(
"PUT", "auth/cert/crls/service-issuer", {"crl": response.text}
)
revoked = json.loads((TRUST / "revoked.json").read_text())
revoked[leaf["serial"]] = True
write_private(
TRUST / "revoked.json", json.dumps(revoked) + "\n", mode=0o644
)
fixture_check("revoked-lb", reject=True)
return {
"revoked_serial": leaf["serial"],
"still_unexpired_at_check": leaf["expires_at"] > time.time(),
"new_tls_connection_denied": True,
"bao_crl_updated": True,
}
def reload_bao():
compose("restart", "openbao")
last = None
for _ in range(60):
operator = Operator()
try:
operator.unseal()
operator.authenticate()
return
except Exception as error:
last = type(error).__name__
finally:
operator.close()
time.sleep(1)
raise TimeoutError("Verified Bao restart/unseal unavailable: " + str(last))
def recover_bao():
# Explicit operator recovery after the server leaf itself has expired.
key, root = ca()
leaf_key, leaf = sign_leaf(
key,
root,
"openbao",
["openbao", "localhost"],
["127.0.0.1"],
seconds=120,
)
install(
"openbao",
{
"certificate": public(leaf).decode(),
"private_key": key_bytes(leaf_key).decode(),
"ca_chain": [public(root).decode()],
},
)
time.sleep(6)
operator = Operator()
try:
operator.unseal()
for role in ROLES:
if role != "migrate":
repair(operator, role)
finally:
operator.close()
def rollover():
old_key, old_root = ca()
new_key, new_root = make_root()
directory = OPERATOR / ("rollover-" + uuid4().hex[:12])
directory.mkdir(mode=0o700)
fixture_key, fixture = sign_leaf(
old_key, old_root, "lb", ["lb"], seconds=900
)
write_private(
SECRET / "probe/retired-ca-lb/cert.pem",
public(fixture) + public(old_root),
mode=0o644,
)
write_private(
SECRET / "probe/retired-ca-lb/key.pem",
key_bytes(fixture_key),
mode=0o644,
)
fixture_check("retired-ca-lb")
for name, value in [
("old-key.pem", key_bytes(old_key)),
("old-ca.pem", public(old_root)),
("new-key.pem", key_bytes(new_key)),
("new-ca.pem", public(new_root)),
]:
write_private(directory / name, value)
journal = {
"phase": "dual_trust",
"old_ca_sha256": old_root.fingerprint(hashes.SHA256()).hex(),
"new_ca_sha256": new_root.fingerprint(hashes.SHA256()).hex(),
}
write_private(directory / "state.json", json.dumps(journal) + "\n")
write_private(
TRUST / "ca.pem", public(old_root) + public(new_root), mode=0o644
)
reload_lb() # Load both roots before any API server leaf changes issuer.
reload_bao() # Native client CA requires an explicit listener restart.
operator = Operator()
try:
for role in [*ROLES, "operator"]:
operator.authorized(
"POST",
"auth/cert/certs/" + role + "-v2",
{
"display_name": role,
"certificate": public(new_root).decode(),
"allowed_common_names": [role],
"allowed_uri_sans": ["spiffe://messenger/" + role],
"token_policies": ["messenger-" + role],
"token_ttl": "900s" if role == "operator" else "120s",
"token_max_ttl": "3600s" if role == "operator" else "600s",
"token_no_default_policy": True,
},
)
csr = operator.authorized(
"POST",
"pki/intermediate/generate/internal",
{
"common_name": "Messenger rollover intermediate "
+ uuid4().hex[:12],
"key_type": "ec",
"key_bits": 256,
},
)["data"]["csr"]
intermediate = sign_intermediate(
new_key, new_root, x509.load_pem_x509_csr(csr.encode())
)
imported = operator.authorized(
"POST",
"pki/intermediate/set-signed",
{
"certificate": (
public(intermediate) + public(new_root)
).decode()
},
)["data"]["imported_issuers"]
chosen = None
for issuer in imported:
details = operator.authorized("GET", "pki/issuer/" + issuer)[
"data"
]
if (
x509.load_pem_x509_certificate(
details["certificate"].encode()
).serial_number
== intermediate.serial_number
):
chosen = issuer
assert chosen
operator.authorized(
"POST",
"pki/config/issuers",
{"default": chosen, "default_follows_latest_issuer": False},
)
journal["phase"] = "new_issuer_active"
write_private(directory / "state.json", json.dumps(journal) + "\n")
deadline = time.monotonic() + 120
active = set(ROLES) - {"migrate"}
while time.monotonic() < deadline:
observed = metadata()
if active <= observed.keys() and all(
observed[r]["issuer"] == intermediate.subject.rfc4514_string()
for r in active
):
break
time.sleep(3)
else:
raise TimeoutError(
"Not all active services renewed into new issuer"
)
fixture_check("retired-ca-lb")
# Allow old L4 connections to expire, then remove old trust everywhere.
time.sleep(46)
for role in [*ROLES, "operator"]:
operator.authorized("DELETE", "auth/cert/certs/" + role + "-v1")
client_key, client_cert = sign_leaf(
new_key, new_root, "operator", ["operator"], seconds=86400
)
write_private(OPERATOR / "client/key.pem", key_bytes(client_key))
write_private(
OPERATOR / "client/cert.pem",
public(client_cert) + public(new_root),
)
write_private(OPERATOR / "ca-key.pem", key_bytes(new_key))
write_private(OPERATOR / "ca.pem", public(new_root))
write_private(TRUST / "ca.pem", public(new_root), mode=0o644)
journal["phase"] = "old_trust_retired"
write_private(directory / "state.json", json.dumps(journal) + "\n")
finally:
operator.close()
reload_lb()
reload_bao()
fixture_check("retired-ca-lb", reject=True)
fixture_check("lb")
journal["phase"] = "complete"
write_private(directory / "state.json", json.dumps(journal) + "\n")
return {
**journal,
"issuer_id": chosen,
"active_service_count": len(active),
"old_l4_connections_waited_seconds": 46,
"old_ca_fixture_valid_during_dual_trust": True,
"old_ca_new_connection_denied_after_retirement": True,
"services": metadata(),
"image_rebuilt": False,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"action", choices=["metadata", "rollover", "recover-bao"]
)
args = parser.parse_args()
if args.action == "metadata":
result = metadata()
elif args.action == "rollover":
result = rollover()
else:
recover_bao()
result = {"recovered": True}
print(json.dumps(result))
if __name__ == "__main__":
main()
+270
View File
@@ -0,0 +1,270 @@
"""Operator-only, idempotent SQL roles and scoped S3 accounts."""
import json
import os
import ssl
import time
import urllib3
from minio import Minio
from minio.credentials import StaticProvider
from minio.minioadmin import MinioAdmin
from psycopg import sql
from scripts.bootstrap_security import OPERATOR, TRUST
from scripts.stand import compose
def attach_policy(client, policy, user):
attached = json.loads(client.user_info(user)).get("policyName", "") or ""
if policy not in attached.split(","):
client.attach_policy([policy], user=user)
def postgres(raw):
# The bootstrap SQL channel is the server's local Unix socket. No password
# or SQL literal is placed in argv; stdin and output are never recorded.
statements = [
"REVOKE ALL ON SCHEMA public FROM PUBLIC; REVOKE ALL ON DATABASE messenger FROM PUBLIC;"
]
grants = {
"api": {
"SELECT": [
"users",
"sessions",
"chats",
"members",
"messages",
"idempotency",
"attachments",
"message_attachments",
"upload_reservations",
"jobs",
"outbox",
"attachment_variants",
"effect_publications",
],
"INSERT": [
"users",
"sessions",
"chats",
"members",
"messages",
"idempotency",
"attachments",
"message_attachments",
"upload_reservations",
"jobs",
"outbox",
],
"UPDATE": ["users", "idempotency", "jobs"],
"DELETE": ["sessions", "upload_reservations"],
},
"worker": {
"SELECT": [
"jobs",
"attachments",
"attachment_variants",
"effect_publications",
"pipeline_faults",
],
"INSERT": [
"attachment_variants",
"effect_publications",
"pipeline_events",
"outbox",
],
"UPDATE": ["jobs", "attachments", "pipeline_faults"],
},
"relay": {
"SELECT": ["jobs", "outbox", "attachments"],
"INSERT": ["pipeline_events", "outbox"],
"UPDATE": ["jobs", "attachments", "outbox"],
},
"observer": {
"SELECT": [
"jobs",
"effect_publications",
"pipeline_events",
"job_slo_observations",
],
"INSERT": ["job_slo_observations"],
"UPDATE": ["job_slo_observations"],
},
"reconciler": {
"SELECT": [
"jobs",
"attachments",
"attachment_variants",
"upload_reservations",
"message_attachments",
"effect_publications",
],
"UPDATE": ["jobs", "attachments"],
"DELETE": [
"attachments",
"attachment_variants",
"upload_reservations",
"jobs",
"outbox",
"job_slo_observations",
],
},
}
# SQL literals use psycopg's own adapter; names are fixed local roles.
for role, permissions in grants.items():
name = "svc_" + role
password = raw["services"][role]["database_password"]
statements.append(
"DO $$ BEGIN IF NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='"
+ name
+ "') THEN CREATE ROLE "
+ name
+ " LOGIN; END IF; END $$;"
)
statements.append(
sql.SQL(
"ALTER ROLE {} WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD {};"
)
.format(sql.Identifier(name), sql.Literal(password))
.as_string()
)
statements.append(
"GRANT CONNECT ON DATABASE messenger TO "
+ name
+ "; GRANT USAGE ON SCHEMA public TO "
+ name
+ ";"
)
statements.append(
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM " + name + ";"
)
statements.append(
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM " + name + ";"
)
for action, tables in permissions.items():
statements.append(
"GRANT "
+ action
+ " ON "
+ ",".join(tables)
+ " TO "
+ name
+ ";"
)
# Only append-only technical event/outbox IDs use sequences.
sequences = {
"api": ["outbox_id_seq"],
"worker": ["outbox_id_seq", "pipeline_events_id_seq"],
"relay": ["outbox_id_seq", "pipeline_events_id_seq"],
}.get(role, [])
if sequences:
statements.append(
"GRANT USAGE ON SEQUENCE "
+ ",".join(sequences)
+ " TO "
+ name
+ ";"
)
statements.append("GRANT EXECUTE ON FUNCTION login_wait(text) TO svc_api;")
result = compose(
"exec",
"-T",
"postgres",
"psql",
"-q",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
input="\n".join(statements),
)
assert "ERROR" not in result
def s3(raw):
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
context.minimum_version = ssl.TLSVersion.TLSv1_3
context.load_cert_chain(
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
)
pool = urllib3.PoolManager(
ssl_context=context,
timeout=urllib3.Timeout(connect=2, read=10),
retries=False,
)
credentials = StaticProvider(raw["s3_root_user"], raw["s3_root_password"])
admin = MinioAdmin(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=credentials,
secure=True,
http_client=pool,
)
client = Minio(
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
credentials=credentials,
secure=True,
http_client=pool,
region="us-east-1",
)
for attempt in range(60):
try:
if not client.bucket_exists("messenger"):
client.make_bucket("messenger")
break
except Exception:
if attempt == 59:
raise RuntimeError("S3 TLS bootstrap unavailable") from None
time.sleep(1)
for role in ["api", "worker", "reconciler"]:
statements = [
{
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger"],
}
]
def allow(actions, resource, condition=None, statements=statements):
statement = {
"Effect": "Allow",
"Action": actions,
"Resource": [resource],
}
if condition:
statement["Condition"] = condition
statements.append(statement)
allow(["s3:GetObject"], "arn:aws:s3:::messenger/objects/*")
if role == "api":
allow(["s3:PutObject"], "arn:aws:s3:::messenger/objects/staging/*")
if role == "worker":
allow(["s3:PutObject"], "arn:aws:s3:::messenger/objects/*/v*/*")
if role == "reconciler":
allow(["s3:DeleteObject"], "arn:aws:s3:::messenger/objects/*")
allow(
["s3:ListBucket"],
"arn:aws:s3:::messenger",
{"StringLike": {"s3:prefix": ["objects/*"]}},
)
values = raw["services"][role]["values"]
admin.policy_add(
"messenger-" + role,
policy={"Version": "2012-10-17", "Statement": statements},
)
admin.user_add(values["S3_ACCESS_KEY"], values["S3_SECRET_KEY"])
attach_policy(admin, "messenger-" + role, values["S3_ACCESS_KEY"])
pool.clear()
def main():
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
postgres(raw)
s3(raw)
print("SQL role grants and scoped S3 accounts prepared; no values printed")
if __name__ == "__main__":
main()
+636
View File
@@ -0,0 +1,636 @@
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
import argparse
import csv
import io
import json
import math
import os
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
from scripts.tls_client import FreshTrustTransport
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, queries, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
SIGNALS = {
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
"queue_depth": "max(messenger_queue_depth)",
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
"worker_busy": "sum(messenger_worker_busy)",
"worker_attempts": "sum(messenger_job_attempts_total)",
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
"collector_queue": "sum(otelcol_exporter_queue_size)",
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
"app_memory": "sum(messenger_process_memory_bytes)",
}
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
def snapshot(phase):
observed = queries(SIGNALS)
values = {}
for key, rows in observed.items():
value = float(rows[0]["value"][1]) if rows else None
values[key] = (
value if value is not None and math.isfinite(value) else None
)
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
return {
"at": time.time(),
"phase": phase,
"signals": values,
"alerts": [
{
"name": a["labels"]["alertname"],
"state": a["state"],
"active_at": a.get("activeAt"),
}
for a in alerts
],
}
def firing(samples):
return {
a["name"]
for sample in samples
for a in sample["alerts"]
if a["state"] == "firing"
}
class Workload:
def __init__(self, client, headers, chat):
self.client, self.headers, self.chat = client, headers, chat
self.stop = threading.Event()
self.phase = "warmup"
self.requests = []
self.uploads = []
self.lock = threading.Lock()
self.slots = threading.BoundedSemaphore(16)
self.pool = ThreadPoolExecutor(max_workers=16)
image = io.BytesIO()
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
self.raw = image.getvalue()
self.threads = [
threading.Thread(target=self.schedule_messages),
threading.Thread(target=self.schedule_uploads),
]
def write(self):
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/messages",
headers={
**self.headers,
"Idempotency-Key": uuid4().hex,
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
json={"text": "Synthetic SLO message " + uuid4().hex},
)
status = response.status_code
if status == 201:
identity = response.json()["id"]
except httpx.HTTPError:
pass
finally:
with self.lock:
self.requests.append(
[
started,
phase,
status,
time.time() - started,
trace_id,
identity,
]
)
self.slots.release()
def schedule_messages(self):
next_at = time.monotonic()
while not self.stop.is_set():
if self.slots.acquire(blocking=False):
self.pool.submit(self.write)
else:
with self.lock:
self.requests.append(
[time.time(), self.phase, -1, 0, "", ""]
)
next_at += 0.25
self.stop.wait(max(0, next_at - time.monotonic()))
def schedule_uploads(self):
while not self.stop.is_set():
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/attachments",
params={"filename": "synthetic-slo.png"},
content=self.raw,
headers={
**self.headers,
"Content-Type": "image/png",
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
)
status = response.status_code
if status == 202:
identity = response.json()["id"]
except httpx.HTTPError:
pass
with self.lock:
self.uploads.append(
[started, phase, status, identity, trace_id]
)
self.stop.wait(max(0, 5 - (time.time() - started)))
def start(self):
for thread in self.threads:
thread.start()
def close(self):
self.stop.set()
for thread in self.threads:
thread.join(20)
self.pool.shutdown(wait=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/slo-research-first.json")
parser.add_argument("--minimum-seconds", type=int, default=1800)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"scope": "full30m"
if args.minimum_seconds >= 1800
else "short-development",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [320, 200],
},
"complete": False,
"samples": [],
"phases": [],
}
study_started = time.monotonic()
output.parent.mkdir(parents=True, exist_ok=True)
load = None
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
with httpx.Client(
transport=FreshTrustTransport(),
base_url=BASE,
timeout=8,
trust_env=False,
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
) as client:
ready(timeout=90)
config = json.loads(compose("config", "--format", "json"))
report["container_limits"] = {
name: {
k: int(service[k])
if k in {"mem_limit", "memswap_limit", "scale"}
else service[k]
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
if k in service
}
for name, service in config["services"].items()
}
report["docker_vm"] = json.loads(
command(
[
"docker",
"info",
"--format",
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
]
)
)
report["runtime_versions"] = sorted(
{
row["metric"].get("service_version", "unknown")
for row in queries(
{"versions": "messenger_process_memory_bytes"}
)["versions"]
}
)
user, _, session = actor(client)
headers = auth(session)
response = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic SLO experiment",
"member_ids": [user["id"]],
},
)
assert response.status_code == 201
chat = response.json()["id"]
load = Workload(client, headers, chat)
workload = load
def save():
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
def hold(name, seconds, *, until_clear=None, max_seconds=None):
load.phase = name
phase = {
"name": name,
"start": time.time(),
"planned_seconds": seconds,
"maximum_seconds": max_seconds or seconds,
}
report["phases"].append(phase)
minimum_end = time.monotonic() + seconds
deadline = time.monotonic() + (max_seconds or seconds)
start_index = len(report["samples"])
while time.monotonic() < deadline:
sample = snapshot(name)
report["samples"].append(sample)
save()
if (
until_clear is not None
and time.monotonic() >= minimum_end
and not until_clear.intersection(firing([sample]))
):
break
time.sleep(min(5, max(0, deadline - time.monotonic())))
phase["end"] = time.time()
phase["firing"] = sorted(firing(report["samples"][start_index:]))
return phase
def db_delay(on):
if on:
sql(
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
)
else:
sql(
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
)
try:
load.start()
warm = hold("warmup", 300)
settled = [
s for s in report["samples"] if s["at"] >= warm["end"] - 60
]
assert not BURN_ALERTS.intersection(firing(settled)), (
"Baseline is not clean"
)
assert (
report["samples"][-1]["signals"]["collector_memory"]
is not None
)
report["baseline"] = snapshot("baseline")
for signal in ["api", "ready"]:
if signal == "api":
activate, restore = (
lambda: db_delay(True),
lambda: db_delay(False),
)
else:
def activate():
return compose("stop", "--timeout", "1", "worker")
def restore():
return compose(
"up", "-d", "--no-deps", "--no-recreate", "worker"
)
brief_started = time.time()
activate()
brief = hold(signal + "_brief", 5)
restore()
settle = hold(
signal + "_brief_recovery", 120 if signal == "api" else 60
)
assert not BURN_ALERTS.intersection(
brief["firing"] + settle["firing"]
), "Brief incident triggered paging"
report.setdefault("short_spikes", []).append(
{
"signal": signal,
"action_started": brief_started,
"paging_events": 0,
}
)
action_started = time.time()
activate()
action_finished = time.time()
degraded = hold(
signal + "_sustained", 180 if signal == "api" else 150
)
restore_started = time.time()
restore()
restore_finished = time.time()
expected = (
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
if signal == "api"
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
)
recovered = hold(
signal + "_recovery",
150 if signal == "api" else 240,
until_clear=expected,
max_seconds=600,
)
assert expected <= set(degraded["firing"]), (
signal,
degraded["firing"],
)
last = report["samples"][-1]
assert not expected.intersection(firing([last])), (
"Incident alerts did not resolve"
)
points = [
s
for s in report["samples"]
if action_finished <= s["at"] <= recovered["end"]
]
timings = {}
for name in sorted(expected):
first = next(
s["at"] for s in points if name in firing([s])
)
resolved = next(
s["at"]
for s in points
if s["at"] >= restore_finished
and name not in firing([s])
)
timings[name] = {
"detect_seconds": round(first - action_finished, 3),
"resolve_seconds": round(
resolved - restore_finished, 3
),
}
report.setdefault("incidents", []).append(
{
"signal": signal,
"action_started": action_started,
"action_finished": action_finished,
"restore_started": restore_started,
"restore_finished": restore_finished,
"timings": timings,
}
)
if signal == "api":
candidates = [
r
for r in load.requests
if action_finished <= r[0] <= restore_started
and r[2] >= 500
]
assert candidates
items = wait(
lambda identity=candidates[0][4]: spans(identity)
)
assert any(
s["name"] == "db.postgresql"
and s["duration_seconds"] >= 2
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in items
)
report["database_diagnostic_trace"] = items
else:
assert (
max(
s["signals"]["oldest_job_age"] or 0 for s in points
)
> 20
)
assert (
max(s["signals"]["queue_depth"] or 0 for s in points)
> 0
)
accepted = [
r
for r in load.uploads
if action_finished <= r[0] <= restore_started
and r[2] == 202
]
assert accepted
report["worker_recovered_trace"] = wait(
lambda identity=accepted[0][4]: spans(identity)
)
report["worker_recovered_attachment"] = accepted[0][3]
outage_before = snapshot("backend_before")
outage_started = time.time()
compose("stop", "--timeout", "1", "loki", "tempo")
outage = hold("backend_unavailable", 90)
points = [
s
for s in report["samples"]
if s["phase"] == "backend_unavailable"
]
assert max(
s["signals"]["collector_failed_spans"] or 0 for s in points
) > (outage_before["signals"]["collector_failed_spans"] or 0)
assert all(
s["signals"]["collector_queue"] is not None
and s["signals"]["collector_queue_capacity"] is not None
and s["signals"]["collector_memory"] is not None
for s in points
)
assert all(
(s["signals"]["collector_queue"] or 0)
<= (s["signals"]["collector_queue_capacity"] or 0)
for s in points
)
assert (
max(s["signals"]["collector_memory"] for s in points)
< report["container_limits"]["otel"]["mem_limit"]
)
assert not BURN_ALERTS.intersection(outage["firing"])
during = [
r for r in load.requests if r[1] == "backend_unavailable"
]
assert during and all(r[2] == 201 for r in during)
before_ready = sql(
"SELECT count(*) FROM attachments WHERE state='ready'"
)
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
wait(lambda: fetch("loki", "/ready", json_result=False))
wait(lambda: fetch("tempo", "/ready", json_result=False))
hold("backend_recovery", 60)
restored = next(
r
for r in reversed(load.uploads)
if r[1] == "backend_recovery" and r[2] == 202
)
wait(
lambda: any(
s["name"] == "attachment.process"
for s in spans(restored[4])
)
)
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
report["backend_outage"] = {
"started": outage_started,
"finished": outage["end"],
"before": outage_before,
"after": snapshot("backend_after"),
"ready_count_before_backend_recovery": int(before_ready),
"successful_writes_during_outage": len(during),
"restored_trace_id": restored[4],
"queue_bounded": True,
"collector_rss_below_limit": True,
"send_failure_counter_counts_attempts_not_unique_loss": True,
}
remaining = args.minimum_seconds - (
time.monotonic() - study_started
)
if remaining > 0:
hold("steady_after_recovery", remaining)
load.close()
load = None
# Read through the public API: a successful SQL insert alone is not
# proof that every acknowledged message remains accessible.
seen, cursor = set(), None
while True:
page = client.get(
f"/api/v1/chats/{chat}/messages",
headers=headers,
params={
"limit": 100,
**({"cursor": cursor} if cursor else {}),
},
)
page.raise_for_status()
data = page.json()
ids = [row["id"] for row in data["items"]]
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
seen.update(ids)
cursor = data.get("next_cursor")
if not cursor:
break
expected_messages = {
r[5] for r in workload.requests if r[2] == 201
}
assert seen == expected_messages
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
for row in accepted_uploads:
def completed(identity=row[3]):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(completed, timeout=120)
for row in accepted_uploads[
:: max(1, len(accepted_uploads) // 10)
]:
response = client.get(
f"/api/v1/attachments/{row[3]}/content", headers=headers
)
assert (
response.status_code == 200
and response.content == workload.raw
)
report["preservation"] = {
"acknowledged_messages": len(expected_messages),
"accessible_messages": len(seen),
"accepted_uploads": len(accepted_uploads),
"all_accepted_uploads_ready": True,
"sampled_originals_match": True,
}
report["rules"] = fetch("prometheus", "/api/v1/rules")
report["final_30m_sli"] = queries(
{
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
}
)
report["complete"] = True
finally:
try:
db_delay(False)
finally:
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"worker",
"loki",
"tempo",
)
if load is not None:
load.close()
report["elapsed_seconds"] = round(
time.monotonic() - study_started, 3
)
request_path = output.with_name(output.stem + "-requests.csv")
with request_path.open("w") as stream:
writer = csv.writer(stream)
writer.writerow(
[
"timestamp",
"phase",
"status",
"seconds",
"trace_id",
"message_id",
]
)
writer.writerows(workload.requests)
report["requests_csv"] = request_path.name
report["request_statuses"] = dict(
Counter(str(r[2]) for r in workload.requests)
)
report["uploads"] = workload.uploads
save()
print(
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
)
if __name__ == "__main__":
main()
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 -m scripts.restart_secure

Some files were not shown because too many files have changed in this diff Show More