WIP: ЛР 7 — БВТ2301 — Маслов — на 5 #26
@@ -18,3 +18,10 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
operator/
|
||||
trust/
|
||||
.work*
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
student-unit/
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
## Лабораторная 7
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Маслов, БВТ2301.
|
||||
Целевая оценка: 5, трек 2 — жизненный цикл сертификатов и секретов.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
**Черновик. Работа пока не готова к сдаче.**
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Перенесено собственное приложение ЛР6: API, очередь задач, обработка изображений отдельным worker, наблюдаемость. Добавлены TLS/mTLS, раздельные удостоверения и права сервисов, конфигурация OpenBao, короткие сертификаты и токены, операторские команды ротации и восстановления, ограничения входа перед хешированием пароля, обновление старого хеша при успешном входе. Границы доверия и ограничения описаны в REPORT.md, contracts/trust-matrix.md и security/THREAT_MODEL.md.
|
||||
|
||||
Пройдены 29 unit-тестов, проверки структуры/OpenAPI/собственного Python-кода и конфигурации Compose. Первый вариант образов собран. pip-audit проверил 40 закреплённых runtime-зависимостей Python без известных уязвимостей по своей базе на момент запуска. Это не проверка безопасности образов.
|
||||
|
||||
Не завершены: запуск всего защищённого стенда, HTTP-регрессия, положительные и отрицательные проверки mTLS/прав, исследование ротации/отзыва/истечения сертификатов и замены CA под нагрузкой, проверка образов и воспроизведение из чистого клона. Пока нет результатов, подтверждающих выполнение требований на оценку 5.
|
||||
|
||||
Запуск оператора: `uv sync --locked`, `make up PYTHON=.venv/bin/python`. Предполагаемый API: `https://localhost:8443`; доверенный CA создаётся локально в `trust/ca.pem`. Команды и подробности: [REPORT.md](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab7/src/branch/solution/bvt2301-maslov/REPORT.md).
|
||||
|
||||
Источники: задание, собственная ЛР6, официальная документация OpenBao/HAProxy/OWASP. Использована помощь ИИ при проектировании, реализации, проверках и подготовке отчёта; чужие студенческие решения не использовались.
|
||||
|
||||
- [ ] Весь обязательный минимум подтверждён запуском и регрессией.
|
||||
- [x] Контракт и публичные тесты сохранены.
|
||||
- [ ] make test и сценарии безопасности выполнены на защищённом стенде.
|
||||
- [ ] Исследование выбранного трека выполнено и измерения приложены.
|
||||
- [x] Секреты и приватные ключи находятся вне Git.
|
||||
- [x] README задания сохранён, собственный REPORT.md заполнен.
|
||||
|
||||
@@ -30,3 +30,9 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
operator/
|
||||
trust/
|
||||
.work*
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
FROM library/golang:1.27.4-alpine@sha256:70bd7ddea8b6cbe5d6943cde279d6b67feb10cc924fb9100d1d8757c1575b280 AS gateway
|
||||
WORKDIR /source
|
||||
COPY infra/tls/*.go ./
|
||||
RUN CGO_ENABLED=0 GOMAXPROCS=2 go build -trimpath -ldflags="-s -w" -o /out/tls-gateway gateway.go
|
||||
|
||||
FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
|
||||
COPY --from=uv /uv /usr/local/bin/uv
|
||||
WORKDIR /app
|
||||
ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
|
||||
ARG VCS_REF=unknown
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY --from=gateway /out/tls-gateway /usr/local/bin/tls-gateway
|
||||
COPY src /app/src
|
||||
COPY migrations /app/migrations
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"]
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "src"]
|
||||
@@ -31,3 +31,20 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
UV ?= uv
|
||||
CA_FILE = trust/ca.pem
|
||||
.PHONY: bootstrap up down lint unit test-student
|
||||
bootstrap:
|
||||
$(UV) run --locked python -m scripts.bootstrap_security
|
||||
up:
|
||||
$(UV) run --locked python -m scripts.up_security --build
|
||||
down:
|
||||
docker compose down
|
||||
lint:
|
||||
$(UV) run --locked ruff check src student-tests student-unit scripts --exclude scripts/check_contract.py --exclude scripts/validate_openapi.py
|
||||
$(UV) run --locked ruff format --check src student-tests student-unit scripts --exclude scripts/check_contract.py --exclude scripts/validate_openapi.py
|
||||
unit:
|
||||
$(UV) run --locked python -m pytest student-unit -q
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" $(UV) run --locked python -m pytest student-tests
|
||||
|
||||
@@ -1,37 +1,40 @@
|
||||
# Отчёт — лабораторная 7
|
||||
# Лабораторная 7 — Маслов, БВТ2301
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Цель — оценка 5, трек 2: жизненный цикл PKI и секретов. Собственный код ЛР6 перенесён поверх независимого skeleton ЛР7. README/COURSE/публичные tests/OpenAPI/check/validate сохранены; trust matrix заполнена как требует задание.
|
||||
|
||||
## Запуск из чистого clone
|
||||
Текущий статус: реализация готовится к первому запуску. Пройдены 29 unit tests, offline/OpenAPI/lint, проверка конфигурации Compose. Сканирование pip-audit2.10.1 проверило 40 закреплённых runtime зависимостей и не нашло известных уязвимостей в своей базе на момент запуска (`evidence/dependency-scan-first.json`). Это не результат проверки образов или полного стенда. HTTP/mTLS/OpenBao/ротация/исследование/чистый clone пока не подтверждены.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
## Запуск оператора
|
||||
|
||||
## Архитектура и решения
|
||||
Нужны Docker Desktop/Compose v2, Git и uv0.13.7. `uv sync --locked`, затем `make up PYTHON=.venv/bin/python`. Команда создаёт локальные CA/secrets, собирает образы и явно выполняет init/unseal/configure. Повторный `make up` сохраняет существующие ключи, пароли и состояние. Для обычного рестарта API операторский init/unseal не нужен: отдельные role agents продолжают renewal.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
API `https://localhost:8443`, Grafana `https://localhost:3447`, S3 operator endpoint `https://localhost:9007`, OpenBao `https://localhost:8207`. Все host binds только 127.0.0.1. Grafana admin password — локальный `secrets/grafana_password`; операторские файлы находятся в `operator/`, не в Git. Проверка API: `make test test-security CA_FILE=trust/ca.pem PYTHON=.venv/bin/python`; полный собственный HTTP suite: `make test-student`; матрица: `uv run --locked python -m scripts.verify_security`.
|
||||
|
||||
## Доказательства критериев
|
||||
Остановка `docker compose down` сохраняет named volumes. Не удалять operator/ и volumes по отдельности: CA, unseal shares и сохранённый Bao должны соответствовать друг другу. `python -m scripts.bao_operator status/unseal` использует проверяемый TLS и operator client cert; токены/shares не печатаются. При длительном seal/expiry требуется явный operator recovery, а не отключение TLS.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
## Архитектура и границы
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
TLS1.3, CA/SAN, mTLS с exact URI, раздельные роли БД/S3/Valkey и scoped cert auth/KV/PKI в OpenBao. Полное покрытие, порты, TTL, локальные transport boundaries и оправдание исключений: [trust matrix](contracts/trust-matrix.md). Угрозы и отрицательные проверки: [THREAT_MODEL](security/THREAT_MODEL.md). Конфиги gateway — `infra/tls`; политики создаёт `scripts/bao_operator.py`.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Root CA private key остаётся у оператора; intermediate key создаётся внутри persistent non-dev OpenBao. Auth tokens короткие, каждое приложение получает только собственные secrets, named volumes разделены по ролям. Snapshot secret имеет конечный срок и проверяется на API request/получении DB connection. S3 credentials читаются при каждой операции; DB conninfo — при новом подключении. Это статические рабочие PG/S3 accounts с управляемой ротацией, не заявляется динамический DB plugin. Сетевой доступ дополнительно требует короткого клиентского cert.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Пароли Argon2id: t3/m65536KiB/p4, 2 KDF slots/API. При успешном входе старый hash обновляется через compare-and-swap; неверный пароль не запускает rehash. SQL limiter общий для реплик: global burst8/refill2/s и account8/min, 429/Retry-After перед KDF. Хеши/session digest/credentials не экспортируются. JSON ограничен 64KiB/5с приёма; изображения и upload-worker сохраняют пределы ЛР5–6.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Наблюдаемость и устойчивый pipeline сохранены: API принимает файл и jobs/outbox устойчиво, отдельный relay/worker, версии и fencing, конечные retries, ручной replay и cleanup. Collector сохраняет bounded queues и limiter; Grafana provisioning, три persistent хранилища и SLO rules перенесены. Полные повторные проверки ещё предстоят; результаты ЛР6 не выдаются за результаты ЛР7.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Исследование на 5
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Гипотеза: автоматическое обновление leaf и service tokens позволяет использовать короткие TTL; dual trust даёт безопасно заменить CA, а expiry/revocation приводят к ограниченному отказу без plaintext fallback. План: фиксированная синтетическая нагрузка, renewal, ротация S3 secret и leaf без image rebuild, отзыв старого leaf с новым TLS подключением, expiry/seal и восстановление, CA rollover с retirement старого trust. Измеряются ошибки, время восстановления, сохранность всех подтверждённых сообщений/заданий и отсутствие утечек. Результаты и ограничения заполняются после реального опыта.
|
||||
|
||||
## Ответы на контрольные вопросы
|
||||
|
||||
1. TLS шифрует поток и проверяет сервер через CA и SAN. В mTLS сервер также проверяет клиентский сертификат. Gateway затем сопоставляет URI с разрешённой ролью, а SQL/S3/broker/Bao ACL ограничивает конкретное действие. Проверка сертификата и разрешение операции — отдельные шаги.
|
||||
2. CA может выдать удостоверение worker, API и LB. Подпись подтверждает доверенного издателя, но worker не становится LB: API принимает только URI lb. Поэтому нужен отдельный тест с чужой ролью, подписанной правильным CA.
|
||||
3. Первое удостоверение создаёт локальный оператор из offline CA. Только собственный agent получает bootstrap leaf; приложения не получают CA private key/root token/unseal. Unseal выполняет оператор двумя shares из трёх защищённых файлов. В учебном стенде это один оператор, без имитации независимых хранителей.
|
||||
4. Пока agent имеет действующий cert/token и Bao доступен, он обновляет удостоверение и секреты. При недоступном/sealed Bao обновление прекращается; cache имеет конечный срок и затем API/новые подключения отказывают. Обычный рестарт API использует продолжающий работать agent. После длительного истечения всех удостоверений оператор восстанавливает trust через явно описанный recovery.
|
||||
|
||||
## Источники и ограничения
|
||||
|
||||
Задание и собственная ЛР6; [OpenBao cert auth](https://openbao.org/docs/auth/cert/), [TLS listener](https://openbao.org/docs/configuration/listener/tcp/), [политики](https://openbao.org/docs/concepts/policies/), [OWASP password storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [HAProxy3.2](https://docs.haproxy.org/3.2/configuration.html). Точные versions/digests находятся в Compose/lock. При проектировании, реализации, проверках и подготовке отчёта использована помощь ИИ; чужие студенческие решения/истории/измерения не использовались.
|
||||
|
||||
Один локальный host и одиночные PG/S3/broker/OpenBao — ограничения доступности. Нет защиты от host root/потери диска. Короткие TTL демонстрационные. Финальные runtime результаты, image scan/triage и воспроизводимость не подтверждаются одним наличием конфигов.
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
PIPELINE_FAULTS: '1'
|
||||
relay:
|
||||
environment:
|
||||
PIPELINE_FAULTS: '1'
|
||||
worker:
|
||||
environment:
|
||||
PIPELINE_FAULTS: '1'
|
||||
controller:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
pull_policy: never
|
||||
entrypoint:
|
||||
- python
|
||||
user: 10001:10001
|
||||
environment:
|
||||
IDENTITY_ENABLED: '1'
|
||||
volumes:
|
||||
- ./secrets/operator-task:/run/identity:ro
|
||||
- ./trust:/trust:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=8m
|
||||
cpus: 0.5
|
||||
mem_limit: 192m
|
||||
pids_limit: 64
|
||||
healthcheck:
|
||||
disable: true
|
||||
command:
|
||||
- -c
|
||||
- import time;time.sleep(86400)
|
||||
+1563
File diff suppressed because it is too large
Load Diff
+33
-28
@@ -1,37 +1,42 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия — Маслов, БВТ2301
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Один локальный стенд, отдельная сеть Compose. В сетевых соединениях используется TLS 1.3 с проверкой цепочки и DNS/IP SAN. Удостоверение сервиса — URI `spiffe://messenger/<роль>`. Gateway проверяет текущий CA, срок сертификата, список отозванных серийных номеров и разрешённую роль; HTTP дополнительно ограничен методом и путём. Конфигурации находятся в `infra/tls/*.json`. Наличие конфигурации не заменяет проверку: фактические результаты должны быть в `evidence/security-*.json` и REPORT.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Связь | Endpoint и защита | Разрешение и срок | Проверка |
|
||||
|---|---|---|---|
|
||||
| Клиент → LB | `https://localhost:8443`, CA `trust/ca.pem`, SAN localhost/127.0.0.1 | Bearer, сессия 3600 с; общий SQL limiter до Argon2 | `make test test-security CA_FILE=trust/ca.pem`; HTTP suite |
|
||||
| LB → все API | `api:8443`, mTLS; DNS SAN api; только URI lb | Роль lb, без обхода на API:8080; leaf 120 с | `python -m scripts.verify_security`: все адреса DNS API, правильный / отсутствующий / чужой CA / другая доверенная роль |
|
||||
| API/worker/relay/observer/reconciler → PostgreSQL | `postgres:5443`, PG SSLRequest + mTLS, `sslmode=verify-full`, `gssencmode=disable` | Разные `svc_<роль>`, SCRAM, без SUPERUSER/CREATEROLE; нет доступа worker к users/sessions; TLS соединение ≤45 с | operator probe: SELECT 1 и запрещённые SQL; `scripts/prepare_dependencies.py` |
|
||||
| API/worker/reconciler → S3 | `https://s3:9443`, mTLS + отдельные SigV4 accounts | API: PUT staging/GET objects; worker: GET objects/PUT versioned; reconciler: LIST/GET/DELETE objects; нет admin | operator probe: разрешённые PUT/GET, запрещённые prefix/DELETE/admin; собственные тестовые объекты удаляются |
|
||||
| API → broker | N/A: API устойчиво пишет jobs/outbox в PG; producer — отдельный relay | API не получает broker password и не входит в allowlist broker | API-cert → broker должен быть отклонён |
|
||||
| Relay/worker → broker | `broker:6380`, mTLS; разные URI relay/worker + разные Valkey ACL | relay XADD/WAITAOF; worker XGROUP/XREADGROUP/XAUTOCLAIM/XACK/XDEL; нет CONFIG; соединение ≤45 с | operator probe: PING после auth, CONFIG denied, три отрицательных TLS случая |
|
||||
| API/worker/relay/observer → Collector | `otel:4438`, mTLS; только перечисленные URI | Только POST `/v1/`; отдельные идентичности API и worker | operator probe: реальный OTLP POST и неверные cert; pipeline в `infra/otel/collector.yaml` |
|
||||
| Collector → Loki/Tempo | `loki:3310/otlp`, `tempo:4438/v1/traces`, mTLS, URI otel | Только ingest; Grafana не может писать ingest | Gateway allowlist и проверки матрицы; bounded exporter queues сохранены |
|
||||
| Grafana → Prometheus/Loki/Tempo | `prometheus:9900`, `loki:3310`, `tempo:3320`, mTLS, URI grafana | Запросы; нет ingest/операторских изменений | Реальные datasource queries и запрещённые методы |
|
||||
| Prometheus → Collector/LB/Loki/Tempo | `otel:18889`, `otel:18888`, `lb:18404`, `loki:3310`, `tempo:3320`, mTLS | Только GET metrics; URI prometheus | Prometheus up/actual series и negative route tests |
|
||||
| Role agents → OpenBao | `https://openbao:8200`, native mTLS + cert auth с exact CN и URI | Только own KV v2 path и own PKI role. Auth token 120 с, max 600 с; bounded KV snapshot ≤60 с; рабочий leaf 120 с | operator probe: own secret 200, чужой secret/policy write/foreign PKI 403; renewal и expiry research |
|
||||
| Оператор → OpenBao/Grafana/S3 | localhost `8207`/`3447`/`9007`, проверяемый TLS; Bao/S3 также operator client cert | Scoped operator policy; Grafana Basic; S3 отдельный root account только у оператора | CLI оператора; app/worker не монтируют operator/ и admin credentials |
|
||||
| API → cache | `cache:6380`, mTLS URI api + Valkey ACL | Только GET/SET `user:*`, TTL карточки 30 с; нет sessions/passwords в cache | TLS/ACL probe, публичные бизнес-тесты |
|
||||
| Репликация БД/S3/broker | N/A: одиночные PG/S3/broker с persistent volumes | Отказ реплики API переносится; отказ хоста/диска остаётся ограничением | HTTPS failover/persistence; не заявляется HA хранилищ |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
## Закрытая локальная граница
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
Native listener каждого сервиса доступен только на 127.0.0.1 в его network namespace. Gateway разделяет namespace через `network_mode: service:<имя>`; API запускает gateway и Python сервер в одном контейнере. Прямые 8080/5432/6379/9000/4318/8889/9090/3100/3200/3000/8404 проверяются из отдельного контейнера и должны отказывать. На host опубликованы только 8443 и локальные административные TLS endpoints.
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
Часть продуктов умеет native TLS, но единый gateway выбран для одинакового обновления коротких сертификатов и точной проверки URI/отзыва. Локальные HTTP выходы Collector/Grafana/Prometheus и OTLP SDK тоже слушают только 127.0.0.1: gateway на каждом новом сетевом соединении читает текущие сертификат и CA, проверяет SAN и server URI. Это явно ограниченный локальный HTTP hop; сетевые границы шифруются. Публичного HTTP fallback нет. PostgreSQL native Unix socket остаётся только для operator bootstrap внутри контейнера PG.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
## Bootstrap, renewal и отзыв
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
`make up` выполняет явные шаги оператора. Root CA private key находится в `operator/` (0700, файлы 0600) и не монтируется в контейнеры. Root CA — 365 дней; intermediate — 30 дней, его ключ создаётся внутри OpenBao. Bootstrap cert сервиса — 30 минут, доступен только собственному agent; рабочий ключ сохраняется в собственном named volume (0700/0600). Обычный рестарт API использует продолжавший работать отдельный agent.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
OpenBao работает с persistent file storage, вне dev-mode. Init создаёт 3 shares, threshold 2; файлы остаются у оператора, не монтируются в Bao/apps. В учебном стенде один оператор хранит все три защищённых файла; разделение между независимыми держателями не моделируется. Первичный root token используется только для установки auth/policies/PKI/KV/audit, затем отзывается и удаляется. У оператора cert-auth с отдельной policy и коротким token.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
Каждая роль получает только собственные KV и PKI разрешения. Agent опрашивает Bao каждые 15 с, обновляет leaf до истечения, сохраняет пары через атомарную смену каталога; snapshot secrets имеет конечный срок. После expiry запросы/новые подключения завершаются контролируемым отказом. Длительная остановка/seal Bao требует явного operator recovery; статического бессрочного секрета для автоматического обхода нет.
|
||||
|
||||
Сертификат сервера Bao тоже короткий и обновляется native `tls_auto_reload`; CA bundle при rollover перечитывается через явный restart/unseal. HAProxy перечитывает leaf через master-worker SIGUSR2, без пересборки. Отзыв проверяется новым TLS подключением; HTTP закрывает соединение после ответа, PG/broker ограничены 45 с и сроком клиентского cert. Список serial deny не подменяет отзыв token/рабочего пароля при компрометации всей роли: эти действия оператор выполняет отдельно.
|
||||
|
||||
## Права файлов и исключения
|
||||
|
||||
Локальная `secrets/` имеет 0700. Отдельные bind files читаемы сервисным UID Docker Desktop; каждому agent смонтирован только его bootstrap каталог. Рабочие keys/snapshots в named volumes имеют 0600 и читаются собственным UID. App/worker — UID10001, read_only, drop ALL, no-new-privileges, bounded tmpfs/pids/CPU/RAM. Stateful данные находятся в отдельных томах.
|
||||
|
||||
Одноразовый identity-init использует UID0 и только CHOWN/DAC_OVERRIDE для начальной подготовки томов; после завершения не работает. Официальный PG entrypoint использует ограниченный набор прав для init/drop-uid. Operator probe запускается только явно с profile operator, read_only и DAC_READ_SEARCH для чтения проверяемых service identities; это инструмент оператора, в приложение он не встраивается и root CA/unseal туда не передаются. Docker socket, host network и privileged отсутствуют.
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"scanner": "pip-audit 2.10.1",
|
||||
"scope": "locked runtime dependencies, no dev group",
|
||||
"lock_sha256": "0fb0fa939296df200ddeda2ad6379a9f1e99ff08fbcfc0ec8020b2a1115cf820",
|
||||
"scanned_dependencies": 40,
|
||||
"vulnerable_dependencies": [],
|
||||
"scan_completed": true,
|
||||
"recorded_at": "2026-11-20T15:08:38.146220+00:00"
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
|
||||
// go mod download validates its h1 checksum through the Go checksum database.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const version = "v0.0.0-20251015172955-9e49d5e7a648"
|
||||
const total int64 = 25342606
|
||||
const chunk int64 = 512 * 1024
|
||||
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
|
||||
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
|
||||
if err := os.MkdirAll(directory, 0755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
target := filepath.Join(directory, version+".zip")
|
||||
if info, err := os.Stat(target); err == nil && info.Size() == total {
|
||||
fmt.Println("Source ZIP cached; Go will validate checksum")
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 90 * time.Second}
|
||||
jobs := make(chan int64)
|
||||
errors := make(chan error, 64)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < 8; worker++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for start := range jobs {
|
||||
end := min(start+chunk, total) - 1
|
||||
part := fmt.Sprintf("%s.part-%d", target, start)
|
||||
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
|
||||
continue
|
||||
}
|
||||
var last error
|
||||
for attempt := 0; attempt < 4; attempt++ {
|
||||
req, _ := http.NewRequest("GET", source, nil)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
last = fmt.Errorf("source range request failed")
|
||||
continue
|
||||
}
|
||||
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
|
||||
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
|
||||
response.Body.Close()
|
||||
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
|
||||
response.Body.Close()
|
||||
if err != nil || int64(len(data)) != end-start+1 {
|
||||
last = fmt.Errorf("incomplete source range")
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(part, data, 0644); err != nil {
|
||||
last = err
|
||||
continue
|
||||
}
|
||||
last = nil
|
||||
fmt.Printf("Source part %d verified\n", start/chunk+1)
|
||||
break
|
||||
}
|
||||
if last != nil {
|
||||
errors <- last
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
jobs <- start
|
||||
}
|
||||
close(jobs)
|
||||
group.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
panic(err)
|
||||
}
|
||||
out, err := os.Create(target + ".partial")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for start := int64(0); start < total; start += chunk {
|
||||
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := io.Copy(out, part); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
part.Close()
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := os.Rename(target+".partial", target); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,789 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Мессенджер: диагностика и SLO",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"timezone": "utc",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"lab6",
|
||||
"slo"
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Запросы через балансировщик, в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "все бизнес-запросы",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Health requests use a separate backend and are excluded. Includes ingress 503 with no API available.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Доля ответов 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[1m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy=\"messenger\"}[1m])),0.001)",
|
||||
"refId": "A",
|
||||
"legendFormat": "ошибки 5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API: p50 / p95 / p99, секунды",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.5, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "B",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum by(le) (rate(messenger_http_duration_seconds_bucket[2m])))",
|
||||
"refId": "C",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Quantiles from combined histogram buckets; complete ASGI response, no average of replica quantiles.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Очередь: ожидающие задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"refId": "A",
|
||||
"legendFormat": "ожидают",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Logical PostgreSQL jobs in queued state, not count of duplicate stream notifications.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Возраст старейшего незавершённого задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(messenger_oldest_job_age_seconds)",
|
||||
"refId": "A",
|
||||
"legendFormat": "возраст",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Обработчик: занято / доступно",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"refId": "A",
|
||||
"legendFormat": "занято",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"refId": "B",
|
||||
"legendFormat": "всего мест",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Соединения PostgreSQL",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name) (messenger_db_pool_used)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}} занято",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum by(service_name) (messenger_db_pool_waiting)",
|
||||
"refId": "B",
|
||||
"legendFormat": "{{service_name}} ожидают",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Попытки обработки по результату",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(outcome) (rate(messenger_job_attempts_total[2m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{outcome}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Длительность попытки обработки: p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "От приёма до ready: p50 / p95 / p99",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.5,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "A",
|
||||
"legendFormat": "p50",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "B",
|
||||
"legendFormat": "p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_time_to_ready_seconds_bucket[2m])))",
|
||||
"refId": "C",
|
||||
"legendFormat": "p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Only completed publications have a duration. Deadline SLI below also counts unfinished tasks.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "SLO API за 30 минут: цель 99%",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\",code=\"5xx\"}[30m])) / clamp_min(sum(increase(haproxy_backend_http_responses_total{proxy=\"messenger\"}[30m])),1)",
|
||||
"refId": "A",
|
||||
"legendFormat": "доступность",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "SLO ready за 30 минут: 99% не позже 10 секунд",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(messenger_slo_decisions_total{outcome=\"good\"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)",
|
||||
"refId": "A",
|
||||
"legendFormat": "доля вовремя",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "Durable one-time decisions per non-obsolete logical job; unfinished jobs become bad at ten seconds. Synthetic 320x200 profile.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "CPU процессов, ядра",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(service_name)(rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Память процессов",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by(service_name)(messenger_process_memory_bytes)",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{service_name}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Буфер экспорта Collector",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "otelcol_exporter_queue_size",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "otelcol_exporter_queue_capacity",
|
||||
"refId": "B",
|
||||
"legendFormat": "{{exporter}} capacity",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Ошибки отправки трасс и логов",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "spans/s",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
|
||||
"refId": "B",
|
||||
"legendFormat": "logs/s",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Состояния локальных alerts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"refId": "A",
|
||||
"legendFormat": "{{alertname}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Связанные логи API и обработчика",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
|
||||
"queryType": "range",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
}
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending"
|
||||
},
|
||||
"description": "Filter structured metadata trace_id in Explore; open the same ID in the Traces datasource."
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: Laboratory 6
|
||||
type: file
|
||||
disableDeletion: true
|
||||
editable: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Metrics
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://127.0.0.1:13909
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
timeInterval: 5s
|
||||
- name: Logs
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://127.0.0.1:13310
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
url: $${__value.raw}
|
||||
datasourceUid: tempo
|
||||
- name: Traces
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://127.0.0.1:13320
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
@@ -0,0 +1,55 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 512
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.3 no-tls-tickets
|
||||
ssl-default-server-options ssl-min-ver TLSv1.3 no-tls-tickets
|
||||
hard-stop-after 10s
|
||||
mworker-max-reloads 2
|
||||
stats socket /tmp/admin.sock mode 600 level admin
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
log-format "http status=%ST duration_ms=%Ta"
|
||||
timeout connect 1s
|
||||
timeout client 15s
|
||||
timeout server 8s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
retries 0
|
||||
option httpclose
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8443 ssl crt /run/identity/current/bundle.pem
|
||||
use_backend health if { path_beg /health/ }
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4
|
||||
server-template api 1-8 api:8443 ssl verify required ca-file /trust/ca.pem crt /run/identity/current/bundle.pem sni str(api) check-sni api verifyhost api no-ssl-reuse
|
||||
|
||||
backend health
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
|
||||
server-template api 1-8 api:8443 ssl verify required ca-file /trust/ca.pem crt /run/identity/current/bundle.pem sni str(api) check-sni api verifyhost api no-ssl-reuse
|
||||
|
||||
frontend internal_metrics
|
||||
bind 127.0.0.1:8404
|
||||
http-request use-service prometheus-exporter if { path /metrics }
|
||||
http-request deny
|
||||
@@ -0,0 +1,14 @@
|
||||
disable_mlock = true
|
||||
ui = false
|
||||
api_addr = "https://openbao:8200"
|
||||
storage "file" { path = "/data/storage" }
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_min_version = "tls13"
|
||||
tls_cert_file = "/run/identity/current/cert.pem"
|
||||
tls_key_file = "/run/identity/current/key.pem"
|
||||
tls_client_ca_file = "/trust/ca.pem"
|
||||
tls_require_and_verify_client_cert = true
|
||||
tls_auto_reload = true
|
||||
tls_auto_reload_interval = "5s"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/api" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/api" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/broker" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/cache" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/grafana" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/lb" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/loki" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/migrate" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/migrate" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/observer" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/observer" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/openbao" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,11 @@
|
||||
path "pki/*" { capabilities = ["create", "read", "update", "delete", "list", "sudo"] }
|
||||
path "auth/cert/*" { capabilities = ["create", "read", "update", "delete", "list"] }
|
||||
path "secret/*" { capabilities = ["create", "read", "update", "delete", "list"] }
|
||||
path "sys/policies/acl/messenger-*" { capabilities = ["create", "read", "update", "delete", "list"] }
|
||||
path "sys/mounts/pki/tune" { capabilities = ["read", "update", "sudo"] }
|
||||
path "sys/seal" { capabilities = ["update", "sudo"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/accessors" { capabilities = ["list", "sudo"] }
|
||||
path "auth/token/lookup-accessor" { capabilities = ["update", "sudo"] }
|
||||
path "auth/token/revoke-accessor" { capabilities = ["update", "sudo"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/otel" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/postgres" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/prometheus" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/reconciler" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/reconciler" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/relay" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/relay" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/s3" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,3 @@
|
||||
path "pki/issue/tempo" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,4 @@
|
||||
path "pki/issue/worker" { capabilities = ["create", "update"] }
|
||||
path "auth/token/renew-self" { capabilities = ["update"] }
|
||||
path "auth/token/lookup-self" { capabilities = ["read"] }
|
||||
path "secret/data/worker" { capabilities = ["read"] }
|
||||
@@ -0,0 +1,50 @@
|
||||
# Диагностика по сохранённой телеметрии
|
||||
|
||||
Открой dashboard «Мессенджер: диагностика и SLO», временная зона UTC.
|
||||
Сопоставляй время оповещения с метриками, трассами и связанными логами.
|
||||
|
||||
## API errors
|
||||
|
||||
Признак: ApiHighErrorRate, ApiFastBurn или ApiSlowBurn. Числитель доступности:
|
||||
`sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m]))`.
|
||||
Знаменатель — тот же backend со всеми code. Health использует другой backend.
|
||||
Сравни API histogram, DB pool used/waiting и worker busy. В Traces найди
|
||||
ошибочный server span и вложенный db.postgresql или s3.*. Длинная DB-операция
|
||||
с ошибкой и занятым пулом при нормальной очереди указывает на БД. Отсутствие
|
||||
API span при ingress503 может означать отсутствие готовых API. Один5xx не
|
||||
доказывает причину. Через trace_id найди соответствующий Logs record.
|
||||
После исправления проверь новые сообщения, спад5xx, resolved и прежние данные.
|
||||
|
||||
## Queue age
|
||||
|
||||
Признак: QueueLagGrowing, ReadyFastBurn или ReadySlowBurn. Сравни логические
|
||||
queued jobs PG, oldest job age и worker busy/capacity. Upload202 при росте
|
||||
ожидающей очереди и отсутствии новых worker spans указывает на обработчик.
|
||||
Если попытки идут и содержат DB/S3 ошибки, причина может быть в зависимости.
|
||||
Найди путь upload→producer→consumer→S3/DB. Histogram ready содержит только
|
||||
завершения, поэтому отдельно проверь deadline SLI для незавершённых jobs.
|
||||
После восстановления проверь тот же attachment_id, original, спад и resolved.
|
||||
|
||||
## Telemetry backend
|
||||
|
||||
Проверь backend health, Collector queue и accepted/sent/failed счётчики.
|
||||
Failed attempts не равны числу уникально потерянных spans: пакет повторяется.
|
||||
После слива очередей сравни изменение accepted-minus-sent и исходное состояние.
|
||||
SDK queues traces/logs512records, timeout0.75s; Collector memory limiter192MiB,
|
||||
batch≤256, очереди logs/traces по32пакета, один sender и finite retry≤10s.
|
||||
При отказе telemetry может теряться; измеряй память и настоящие бизнес-запросы.
|
||||
После восстановления найди новую трассу и лог, а не только старый screenshot.
|
||||
|
||||
## SLO и окна
|
||||
|
||||
Учебный профиль: синтетические PNG320×200, upload не чаще одного каждые3s,
|
||||
закреплённые CPU/RAM. API цель99%ответов без5xx за30m на ingress. Jobs цель99%
|
||||
ready≤10s от PG created_at принимающей транзакции до committed_at публикации.
|
||||
Начальная граница консервативно включает завершение принимающей транзакции.
|
||||
Однократные durable good/bad решения учитывают failed и незавершённые после10s.
|
||||
Ручной replay не стирает прежний bad; устаревшие до решения jobs исключены.
|
||||
Burn rate = доля плохих событий/0.01. Быстрое правило: burn>14.4 в30s и2m,
|
||||
непрерывно45s. Длительное: burn>6 в2m и5m, непрерывно60s. Короткие окна выбраны
|
||||
для воспроизводимого стенда, а не универсальной production paging policy.
|
||||
Измеряются краткий всплеск/длительная деградация/firing/resolved/ложные события.
|
||||
Точность времени ограничена polling2s, scrape/evaluation5s и экспортом.
|
||||
@@ -0,0 +1,53 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: ApiHighErrorRate
|
||||
expr: sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) > 0.05
|
||||
for: 20s
|
||||
labels: {severity: warning, signal: api}
|
||||
annotations:
|
||||
summary: More than 5 percent business responses are 5xx
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: QueueLagGrowing
|
||||
expr: max(messenger_oldest_job_age_seconds) > 20 and max(messenger_queue_depth) > 0
|
||||
for: 10s
|
||||
labels: {severity: warning, signal: ready}
|
||||
annotations:
|
||||
summary: Oldest unfinished attachment exceeds 20 seconds
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: ApiFastBurn
|
||||
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[30s])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[30s])),0.001) / 0.01 > 14.4) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 14.4)
|
||||
for: 45s
|
||||
labels: {severity: critical, signal: api, burn: fast}
|
||||
annotations:
|
||||
summary: API burns the one-percent error budget in both fast windows
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: ApiSlowBurn
|
||||
expr: (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[2m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[2m])),0.001) / 0.01 > 6) and (sum(rate(haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}[5m])) / clamp_min(sum(rate(haproxy_backend_http_responses_total{proxy="messenger"}[5m])),0.001) / 0.01 > 6)
|
||||
for: 60s
|
||||
labels: {severity: critical, signal: api, burn: slow}
|
||||
annotations:
|
||||
summary: API burns the error budget across two longer windows
|
||||
runbook: infra/otel/RUNBOOK.md#api-errors
|
||||
- alert: ReadyFastBurn
|
||||
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[30s])) / clamp_min(sum(rate(messenger_slo_decisions_total[30s])),0.001) / 0.01 > 14.4) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 14.4)
|
||||
for: 45s
|
||||
labels: {severity: critical, signal: ready, burn: fast}
|
||||
annotations:
|
||||
summary: Attachment deadline SLO burns the error budget
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: ReadySlowBurn
|
||||
expr: (sum(rate(messenger_slo_decisions_total{outcome="bad"}[2m])) / clamp_min(sum(rate(messenger_slo_decisions_total[2m])),0.001) / 0.01 > 6) and (sum(rate(messenger_slo_decisions_total{outcome="bad"}[5m])) / clamp_min(sum(rate(messenger_slo_decisions_total[5m])),0.001) / 0.01 > 6)
|
||||
for: 60s
|
||||
labels: {severity: critical, signal: ready, burn: slow}
|
||||
annotations:
|
||||
summary: Sustained misses of the attachment ten-second deadline
|
||||
runbook: infra/otel/RUNBOOK.md#queue-age
|
||||
- alert: CollectorUnavailable
|
||||
expr: up{job="collector"} == 0
|
||||
for: 15s
|
||||
labels: {severity: warning, signal: telemetry}
|
||||
annotations:
|
||||
summary: Collector self metrics are unavailable
|
||||
runbook: infra/otel/RUNBOOK.md#telemetry-backend
|
||||
@@ -0,0 +1,88 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 127.0.0.1:8889
|
||||
metric_expiration: 15s
|
||||
resource_constant_labels:
|
||||
included:
|
||||
- service.name
|
||||
- service.instance.id
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
otlp_http/tempo:
|
||||
endpoint: http://127.0.0.1:14319
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 32
|
||||
num_consumers: 1
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 10s
|
||||
otlp_http/loki:
|
||||
endpoint: http://127.0.0.1:13310/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
queue_size: 32
|
||||
num_consumers: 1
|
||||
block_on_overflow: false
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 10s
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 127.0.0.1:13133
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 127.0.0.1
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/tempo
|
||||
logs:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/loki
|
||||
metrics:
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
@@ -0,0 +1,44 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
log_level: warn
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2026-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 8
|
||||
volume_enabled: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes:
|
||||
- service.name
|
||||
- action: structured_metadata
|
||||
attributes:
|
||||
- service.instance.id
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,30 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: application_otlp
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:18889
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:18888
|
||||
- job_name: ingress
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:18404
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost:9090
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:13200
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:13100
|
||||
@@ -0,0 +1,22 @@
|
||||
target: all
|
||||
stream_over_http_enabled: true
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"otel": "otel/opentelemetry-collector-contrib:0.165.0@sha256:e40d61f10c9a617cca1ffd1be4d7e2b3a4b0f0001798ff07e58cabe82e618cea",
|
||||
"prometheus": "prom/prometheus:v3.16.0@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06",
|
||||
"loki": "grafana/loki:3.7.10@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7",
|
||||
"tempo": "grafana/tempo:3.1.1@sha256:64e2fbd56d9c2b3412a4b430a58ea59e91a670b1e75f920a0c702213ad145044",
|
||||
"grafana": "grafana/grafana:13.3.1@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf"
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root_user=$(cat /run/secrets/s3_root_user)
|
||||
root_password=$(cat /run/secrets/s3_root_password)
|
||||
api_key=$(cat /run/secrets/s3_access_key)
|
||||
api_secret=$(cat /run/secrets/s3_secret_key)
|
||||
# Credentials are passed only inside this isolated bootstrap container.
|
||||
mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1
|
||||
mc mb --ignore-existing local/messenger >/dev/null 2>&1
|
||||
mc anonymous set none local/messenger >/dev/null 2>&1
|
||||
mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1
|
||||
mc admin policy create local messenger-api /policy.json >/dev/null 2>&1
|
||||
mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1
|
||||
printf '%s\n' 'Private bucket and scoped application policy ready'
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
export MINIO_BROWSER=off
|
||||
exec minio server /data --address 127.0.0.1:9000 --console-address 127.0.0.1:9001
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger/objects/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
|
||||
|
||||
FROM toolchain AS minio-build
|
||||
WORKDIR /source
|
||||
COPY infra/fetch-source.go /tmp/fetch-source.go
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go run /tmp/fetch-source.go && \
|
||||
for attempt in 1 2 3 4; do \
|
||||
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \
|
||||
test "$attempt" -lt 4 || exit 1; sleep 1; \
|
||||
done && \
|
||||
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
for attempt in 1 2 3; do \
|
||||
timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \
|
||||
test "$attempt" -lt 3 || exit 1; sleep 1; \
|
||||
done
|
||||
|
||||
FROM toolchain AS mc-build
|
||||
WORKDIR /source
|
||||
RUN git init . && git remote add origin https://github.com/minio/mc.git && \
|
||||
git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD
|
||||
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
|
||||
go build -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
|
||||
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
|
||||
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
|
||||
RUN mkdir /data && chown 10001:10001 /data
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/s3-entrypoint"]
|
||||
EXPOSE 9000
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
|
||||
|
||||
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
|
||||
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
|
||||
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap
|
||||
COPY infra/s3-policy.json /policy.json
|
||||
USER 10001:10001
|
||||
ENV MC_CONFIG_DIR=/tmp/mc
|
||||
ENTRYPOINT ["/usr/local/bin/s3-bootstrap"]
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"role": "api",
|
||||
"dns": [
|
||||
"api",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "broker",
|
||||
"dns": [
|
||||
"broker"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "cache",
|
||||
"dns": [
|
||||
"cache"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "grafana",
|
||||
"dns": [
|
||||
"grafana",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"role": "lb",
|
||||
"dns": [
|
||||
"lb",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false,
|
||||
"reload_pid_one": true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "loki",
|
||||
"dns": [
|
||||
"loki"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "migrate",
|
||||
"dns": [
|
||||
"migrate"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "observer",
|
||||
"dns": [
|
||||
"observer"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "openbao",
|
||||
"dns": [
|
||||
"openbao",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "otel",
|
||||
"dns": [
|
||||
"otel"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "postgres",
|
||||
"dns": [
|
||||
"postgres"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "prometheus",
|
||||
"dns": [
|
||||
"prometheus"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "reconciler",
|
||||
"dns": [
|
||||
"reconciler"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "relay",
|
||||
"dns": [
|
||||
"relay"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "s3",
|
||||
"dns": [
|
||||
"s3",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "tempo",
|
||||
"dns": [
|
||||
"tempo"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"role": "worker",
|
||||
"dns": [
|
||||
"worker"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": false,
|
||||
"servers": [],
|
||||
"outbounds": [],
|
||||
"bootstrap": "/bootstrap",
|
||||
"kv": true
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"role": "api",
|
||||
"dns": [
|
||||
"api",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:8443",
|
||||
"target": "127.0.0.1:8080",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/lb": [
|
||||
"* /"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:14318",
|
||||
"target": "https://otel:4438",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"role": "broker",
|
||||
"dns": [
|
||||
"broker"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:6380",
|
||||
"target": "127.0.0.1:6379",
|
||||
"protocol": "tcp",
|
||||
"allow": {
|
||||
"spiffe://messenger/relay": [],
|
||||
"spiffe://messenger/worker": [],
|
||||
"spiffe://messenger/operator": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"role": "cache",
|
||||
"dns": [
|
||||
"cache"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:6380",
|
||||
"target": "127.0.0.1:6379",
|
||||
"protocol": "tcp",
|
||||
"allow": {
|
||||
"spiffe://messenger/api": [],
|
||||
"spiffe://messenger/operator": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,631 @@
|
||||
// A bounded TLS boundary; cleartext targets must bind only to loopback.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Boundary struct {
|
||||
Listen string `json:"listen"`
|
||||
Target string `json:"target"`
|
||||
Protocol string `json:"protocol"`
|
||||
// Identity -> allowed HTTP method/path prefixes; an empty list denies HTTP.
|
||||
Allow map[string][]string `json:"allow"`
|
||||
Public bool `json:"public"`
|
||||
}
|
||||
type Outbound struct {
|
||||
Listen string `json:"listen"`
|
||||
Target string `json:"target"`
|
||||
Identity string `json:"server_identity"`
|
||||
}
|
||||
type Config struct {
|
||||
Outbounds []Outbound `json:"outbounds"`
|
||||
ServeOnly bool `json:"serve_only"`
|
||||
ReloadPIDOne bool `json:"reload_pid_one"`
|
||||
Role string `json:"role"`
|
||||
DNS []string `json:"dns"`
|
||||
IP []string `json:"ip"`
|
||||
Bao string `json:"bao"`
|
||||
Trust string `json:"trust"`
|
||||
Bootstrap string `json:"bootstrap"`
|
||||
Identity string `json:"identity"`
|
||||
Revoked string `json:"revoked"`
|
||||
TTL string `json:"ttl"`
|
||||
KV bool `json:"kv"`
|
||||
Servers []Boundary `json:"servers"`
|
||||
}
|
||||
type Agent struct {
|
||||
c Config
|
||||
mu sync.Mutex
|
||||
token string
|
||||
tokenUntil time.Time
|
||||
}
|
||||
|
||||
func event(name string) { log.Print("{\"event\":\"" + name + "\"}") }
|
||||
func trust(c Config) (*x509.CertPool, error) {
|
||||
raw, e := os.ReadFile(c.Trust)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(raw) {
|
||||
return nil, errors.New("invalid trust bundle")
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
func material(c Config) (tls.Certificate, error) {
|
||||
base := filepath.Join(c.Identity, "current")
|
||||
if real, e := filepath.EvalSymlinks(base); e == nil {
|
||||
base = real
|
||||
} else {
|
||||
base = c.Bootstrap
|
||||
}
|
||||
return tls.LoadX509KeyPair(filepath.Join(base, "cert.pem"), filepath.Join(base, "key.pem"))
|
||||
}
|
||||
func revoked(c Config, cert *x509.Certificate) bool {
|
||||
raw, e := os.ReadFile(c.Revoked)
|
||||
if e != nil {
|
||||
return true
|
||||
} // missing policy fails closed
|
||||
var entries map[string]bool
|
||||
if json.Unmarshal(raw, &entries) != nil {
|
||||
return true
|
||||
}
|
||||
return entries[strings.ToLower(cert.SerialNumber.Text(16))]
|
||||
}
|
||||
func peer(c Config, state tls.ConnectionState, b Boundary) (string, error) {
|
||||
if b.Public {
|
||||
return "", nil
|
||||
}
|
||||
if len(state.VerifiedChains) == 0 || len(state.PeerCertificates) == 0 {
|
||||
return "", errors.New("client certificate required")
|
||||
}
|
||||
cert := state.PeerCertificates[0]
|
||||
roots, err := trust(c)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
intermediates := x509.NewCertPool()
|
||||
for _, item := range state.PeerCertificates[1:] {
|
||||
intermediates.AddCert(item)
|
||||
}
|
||||
if _, err = cert.Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
|
||||
return "", errors.New("current trust denies client")
|
||||
}
|
||||
if time.Now().Before(cert.NotBefore) || !time.Now().Before(cert.NotAfter) || revoked(c, cert) {
|
||||
return "", errors.New("client certificate unavailable")
|
||||
}
|
||||
for _, identity := range cert.URIs {
|
||||
if _, ok := b.Allow[identity.String()]; ok {
|
||||
return identity.String(), nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("client identity denied")
|
||||
}
|
||||
func serverTLS(c Config, b Boundary) *tls.Config {
|
||||
return &tls.Config{MinVersion: tls.VersionTLS13, SessionTicketsDisabled: true, GetConfigForClient: func(_ *tls.ClientHelloInfo) (*tls.Config, error) {
|
||||
cert, e := material(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
roots, e := trust(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
conf := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}, ClientCAs: roots, SessionTicketsDisabled: true}
|
||||
if !b.Public {
|
||||
conf.ClientAuth = tls.RequireAndVerifyClientCert
|
||||
conf.VerifyConnection = func(s tls.ConnectionState) error { _, e := peer(c, s, b); return e }
|
||||
}
|
||||
return conf, nil
|
||||
}}
|
||||
}
|
||||
func atomicFile(path string, raw []byte) error {
|
||||
temp := path + ".new"
|
||||
if e := os.WriteFile(temp, raw, 0600); e != nil {
|
||||
return e
|
||||
}
|
||||
return os.Rename(temp, path)
|
||||
}
|
||||
func (a *Agent) request(method, path string, body any, authenticated bool) (map[string]any, error) {
|
||||
cert, e := material(a.c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
roots, e := trust(a.c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{cert}}, DisableKeepAlives: true, ResponseHeaderTimeout: 4 * time.Second}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
|
||||
var encoded []byte
|
||||
if body != nil {
|
||||
encoded, e = json.Marshal(body)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
}
|
||||
request, e := http.NewRequest(method, a.c.Bao+"/v1/"+path, bytes.NewReader(encoded))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if authenticated {
|
||||
request.Header.Set("X-Vault-Token", a.token)
|
||||
}
|
||||
response, e := client.Do(request)
|
||||
if e != nil {
|
||||
return nil, errors.New("identity service unavailable")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("identity service status %d", response.StatusCode)
|
||||
}
|
||||
var data map[string]any
|
||||
e = json.NewDecoder(io.LimitReader(response.Body, 1024*1024)).Decode(&data)
|
||||
return data, e
|
||||
}
|
||||
func (a *Agent) refresh() error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
now := time.Now()
|
||||
if a.token != "" && time.Until(a.tokenUntil) < 95*time.Second {
|
||||
result, err := a.request("POST", "auth/token/renew-self", map[string]string{"increment": "120s"}, true)
|
||||
if err == nil {
|
||||
auth, ok := result["auth"].(map[string]any)
|
||||
if !ok {
|
||||
return errors.New("invalid renewal response")
|
||||
}
|
||||
ttl, ok := auth["lease_duration"].(float64)
|
||||
if !ok || ttl < 1 {
|
||||
return errors.New("invalid renewed lease")
|
||||
}
|
||||
a.tokenUntil = now.Add(time.Duration(ttl * float64(time.Second)))
|
||||
event("identity.token_renewed")
|
||||
} else {
|
||||
a.token = ""
|
||||
}
|
||||
}
|
||||
if a.token == "" {
|
||||
// The exact CN+URI constraints select the correct role. Omitting name
|
||||
// permits old/new CA roles to coexist safely during trust rollover.
|
||||
result, err := a.request("POST", "auth/cert/login", map[string]string{}, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
auth, ok := result["auth"].(map[string]any)
|
||||
if !ok {
|
||||
return errors.New("invalid auth response")
|
||||
}
|
||||
a.token, ok = auth["client_token"].(string)
|
||||
if !ok {
|
||||
return errors.New("missing service token")
|
||||
}
|
||||
ttl, ok := auth["lease_duration"].(float64)
|
||||
if !ok || ttl < 1 {
|
||||
return errors.New("invalid service lease")
|
||||
}
|
||||
a.tokenUntil = now.Add(time.Duration(ttl * float64(time.Second)))
|
||||
event("identity.authenticated")
|
||||
}
|
||||
|
||||
cert, e := material(a.c)
|
||||
_, currentErr := os.Stat(filepath.Join(a.c.Identity, "current"))
|
||||
_, initializedErr := os.Stat(filepath.Join(a.c.Identity, "agent.json"))
|
||||
renew := initializedErr != nil || currentErr != nil || e != nil || cert.Leaf == nil || time.Until(cert.Leaf.NotAfter) < 80*time.Second
|
||||
if renew {
|
||||
request := map[string]any{"common_name": a.c.Role, "alt_names": strings.Join(a.c.DNS, ","), "uri_sans": "spiffe://messenger/" + a.c.Role, "ttl": a.c.TTL}
|
||||
if len(a.c.IP) > 0 {
|
||||
request["ip_sans"] = strings.Join(a.c.IP, ",")
|
||||
}
|
||||
result, e := a.request("POST", "pki/issue/"+a.c.Role, request, true)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
data, ok := result["data"].(map[string]any)
|
||||
if !ok {
|
||||
return errors.New("invalid PKI response")
|
||||
}
|
||||
crt, ok := data["certificate"].(string)
|
||||
if !ok {
|
||||
return errors.New("missing certificate")
|
||||
}
|
||||
key, ok := data["private_key"].(string)
|
||||
if !ok {
|
||||
return errors.New("missing private key")
|
||||
}
|
||||
if chain, ok := data["ca_chain"].([]any); ok {
|
||||
for _, item := range chain {
|
||||
crt += "\n" + item.(string)
|
||||
}
|
||||
}
|
||||
parsed, e := tls.X509KeyPair([]byte(crt), []byte(key))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
dir, e := os.MkdirTemp(a.c.Identity, "generation-")
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = os.WriteFile(filepath.Join(dir, "cert.pem"), []byte(crt), 0600); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = os.WriteFile(filepath.Join(dir, "key.pem"), []byte(key), 0600); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = os.WriteFile(filepath.Join(dir, "bundle.pem"), []byte(crt+"\n"+key), 0600); e != nil {
|
||||
return e
|
||||
}
|
||||
link := filepath.Join(a.c.Identity, "next")
|
||||
os.Remove(link)
|
||||
if e = os.Symlink(filepath.Base(dir), link); e != nil {
|
||||
return e
|
||||
}
|
||||
if e = os.Rename(link, filepath.Join(a.c.Identity, "current")); e != nil {
|
||||
return e
|
||||
}
|
||||
// Retain only the current and immediately previous generations, bounding tmpfs.
|
||||
entries, _ := os.ReadDir(a.c.Identity)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() && strings.HasPrefix(entry.Name(), "generation-") && entry.Name() != filepath.Base(dir) {
|
||||
info, _ := entry.Info()
|
||||
if info != nil && now.Sub(info.ModTime()) > 3*time.Minute {
|
||||
os.RemoveAll(filepath.Join(a.c.Identity, entry.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
if parsed.Leaf == nil {
|
||||
return errors.New("unparsed certificate")
|
||||
}
|
||||
event("identity.certificate_issued")
|
||||
if a.c.ReloadPIDOne {
|
||||
if e = syscall.Kill(1, syscall.SIGUSR2); e != nil {
|
||||
return errors.New("certificate reload failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.c.KV {
|
||||
result, e := a.request("GET", "secret/data/"+a.c.Role, nil, true)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
outer, ok := result["data"].(map[string]any)
|
||||
if !ok {
|
||||
return errors.New("invalid secret response")
|
||||
}
|
||||
values, ok := outer["data"].(map[string]any)
|
||||
if !ok {
|
||||
return errors.New("invalid secret values")
|
||||
}
|
||||
expiry := now.Add(60 * time.Second)
|
||||
if a.tokenUntil.Before(expiry) {
|
||||
expiry = a.tokenUntil
|
||||
}
|
||||
data, e := json.Marshal(map[string]any{"fetched_at": now.Unix(), "valid_until": expiry.Unix(), "values": values})
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if e = atomicFile(filepath.Join(a.c.Identity, "secrets.json"), data); e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
return atomicFile(filepath.Join(a.c.Identity, "agent.json"), []byte(`{"refreshed_at":`+strconv.FormatInt(now.Unix(), 10)+`}`))
|
||||
}
|
||||
|
||||
type boundedListener struct {
|
||||
net.Listener
|
||||
slots chan struct{}
|
||||
}
|
||||
type boundedConn struct {
|
||||
net.Conn
|
||||
once sync.Once
|
||||
release func()
|
||||
}
|
||||
|
||||
func (c *boundedConn) Close() error { e := c.Conn.Close(); c.once.Do(c.release); return e }
|
||||
func (l *boundedListener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
c, e := l.Listener.Accept()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
select {
|
||||
case l.slots <- struct{}{}:
|
||||
return &boundedConn{Conn: c, release: func() { <-l.slots }}, nil
|
||||
default:
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
func runHTTP(c Config, b Boundary) error {
|
||||
target, e := url.Parse("http://" + b.Target)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
proxy.Transport = &http.Transport{MaxIdleConns: 32, MaxIdleConnsPerHost: 16, ResponseHeaderTimeout: 8 * time.Second, IdleConnTimeout: 30 * time.Second}
|
||||
proxy.ErrorLog = log.New(io.Discard, "", 0)
|
||||
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
|
||||
w.Header().Set("Connection", "close")
|
||||
http.Error(w, "Protected service unavailable", 503)
|
||||
}
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Force a fresh handshake after each response: no indefinitely cached
|
||||
// certificate identity across rollover, expiry or serial revocation.
|
||||
w.Header().Set("Connection", "close")
|
||||
identity, e := peer(c, *r.TLS, b)
|
||||
if e != nil {
|
||||
w.Header().Set("Connection", "close")
|
||||
http.Error(w, "Client identity denied", 403)
|
||||
return
|
||||
}
|
||||
allowed := b.Public
|
||||
for _, rule := range b.Allow[identity] {
|
||||
parts := strings.SplitN(rule, " ", 2)
|
||||
if len(parts) == 2 && (parts[0] == "*" || parts[0] == r.Method) && strings.HasPrefix(r.URL.Path, parts[1]) {
|
||||
allowed = true
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
http.Error(w, "Operation denied", 403)
|
||||
return
|
||||
}
|
||||
r.Header.Del("X-Forwarded-Client-Cert")
|
||||
r.Header.Del("X-TLS-Identity")
|
||||
proxy.ServeHTTP(w, r)
|
||||
})
|
||||
listener, e := net.Listen("tcp", b.Listen)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
listener = &boundedListener{Listener: listener, slots: make(chan struct{}, 128)}
|
||||
server := &http.Server{Handler: handler, TLSConfig: serverTLS(c, b), ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 32768, ErrorLog: log.New(io.Discard, "", 0)}
|
||||
return server.Serve(tls.NewListener(listener, server.TLSConfig))
|
||||
}
|
||||
|
||||
// Native clients use a private loopback endpoint when their certificate
|
||||
// reload behavior cannot meet the short credential lifetime. Every network
|
||||
// connection is verified mTLS; configuration cannot publish this HTTP hop.
|
||||
func runOutbound(c Config, b Outbound) error {
|
||||
target, e := url.Parse(b.Target)
|
||||
if e != nil || target.Scheme != "https" || target.Hostname() == "" {
|
||||
return errors.New("outbound TLS target required")
|
||||
}
|
||||
host, _, e := net.SplitHostPort(b.Listen)
|
||||
if e != nil || host != "127.0.0.1" {
|
||||
return errors.New("outbound listener must be loopback")
|
||||
}
|
||||
proxy := httputil.NewSingleHostReverseProxy(target)
|
||||
proxy.Transport = &http.Transport{
|
||||
DisableKeepAlives: true, MaxConnsPerHost: 32, ResponseHeaderTimeout: 8 * time.Second,
|
||||
DialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
cert, e := material(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
roots, e := trust(c)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
config := &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{cert}, ServerName: target.Hostname(), VerifyConnection: func(state tls.ConnectionState) error {
|
||||
if len(state.VerifiedChains) == 0 || len(state.PeerCertificates) == 0 {
|
||||
return errors.New("unverified protected server")
|
||||
}
|
||||
leaf := state.PeerCertificates[0]
|
||||
if revoked(c, leaf) {
|
||||
return errors.New("protected server revoked")
|
||||
}
|
||||
if b.Identity != "" {
|
||||
found := false
|
||||
for _, identity := range leaf.URIs {
|
||||
if identity.String() == b.Identity {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return errors.New("protected server identity denied")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
dialer := tls.Dialer{NetDialer: &net.Dialer{Timeout: 2 * time.Second}, Config: config}
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
},
|
||||
}
|
||||
proxy.ErrorLog = log.New(io.Discard, "", 0)
|
||||
proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, e error) {
|
||||
w.Header().Set("Connection", "close")
|
||||
http.Error(w, "Protected destination unavailable", 503)
|
||||
}
|
||||
listener, e := net.Listen("tcp", b.Listen)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
listener = &boundedListener{Listener: listener, slots: make(chan struct{}, 64)}
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Connection", "close")
|
||||
proxy.ServeHTTP(w, r)
|
||||
})
|
||||
server := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second, MaxHeaderBytes: 32768, ErrorLog: log.New(io.Discard, "", 0)}
|
||||
return server.Serve(listener)
|
||||
}
|
||||
func runTCP(c Config, b Boundary) error {
|
||||
listener, e := net.Listen("tcp", b.Listen)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
slots := make(chan struct{}, 128)
|
||||
for {
|
||||
raw, e := listener.Accept()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
default:
|
||||
raw.Close()
|
||||
continue
|
||||
}
|
||||
go func() {
|
||||
defer func() { <-slots }()
|
||||
defer raw.Close()
|
||||
raw.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if b.Protocol == "postgres" {
|
||||
request := make([]byte, 8)
|
||||
if _, e := io.ReadFull(raw, request); e != nil {
|
||||
return
|
||||
}
|
||||
if binary.BigEndian.Uint32(request[:4]) != 8 || binary.BigEndian.Uint32(request[4:]) != 80877103 {
|
||||
return
|
||||
}
|
||||
if _, e := raw.Write([]byte{'S'}); e != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
client := tls.Server(raw, serverTLS(c, b))
|
||||
if e := client.Handshake(); e != nil {
|
||||
return
|
||||
}
|
||||
target, e := net.DialTimeout("tcp", b.Target, 2*time.Second)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
defer target.Close()
|
||||
// Bound a previously authenticated connection even after expiry/revocation.
|
||||
deadline := time.Now().Add(45 * time.Second)
|
||||
state := client.ConnectionState()
|
||||
if len(state.PeerCertificates) > 0 && state.PeerCertificates[0].NotAfter.Before(deadline) {
|
||||
deadline = state.PeerCertificates[0].NotAfter
|
||||
}
|
||||
client.SetDeadline(deadline)
|
||||
target.SetDeadline(deadline)
|
||||
done := make(chan struct{}, 1)
|
||||
go func() { io.Copy(target, client); done <- struct{}{} }()
|
||||
io.Copy(client, target)
|
||||
client.Close()
|
||||
target.Close()
|
||||
<-done
|
||||
}()
|
||||
}
|
||||
}
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
event("gateway.invalid_configuration")
|
||||
os.Exit(2)
|
||||
}
|
||||
raw, e := os.ReadFile(os.Args[1])
|
||||
if e != nil {
|
||||
event("gateway.invalid_configuration")
|
||||
os.Exit(2)
|
||||
}
|
||||
var c Config
|
||||
if json.Unmarshal(raw, &c) != nil || c.Role == "" {
|
||||
event("gateway.invalid_configuration")
|
||||
os.Exit(2)
|
||||
}
|
||||
if c.TTL == "" {
|
||||
c.TTL = "120s"
|
||||
}
|
||||
if e = os.MkdirAll(c.Identity, 0700); e != nil {
|
||||
event("gateway.identity_directory_unavailable")
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, b := range c.Servers {
|
||||
host, _, err := net.SplitHostPort(b.Target)
|
||||
if err != nil || (host != "127.0.0.1" && host != "[::1]" && host != "::1") {
|
||||
event("gateway.nonlocal_target_denied")
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
agent := &Agent{c: c}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
// Bootstrap failures never start a cleartext fallback listener.
|
||||
if !c.ServeOnly {
|
||||
for {
|
||||
if e = agent.refresh(); e == nil {
|
||||
break
|
||||
}
|
||||
event("identity.refresh_failed")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if agent.refresh() != nil {
|
||||
event("identity.refresh_failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
} else {
|
||||
for {
|
||||
cert, err := material(c)
|
||||
if err == nil && cert.Leaf != nil && time.Now().Before(cert.Leaf.NotAfter) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, boundary := range c.Servers {
|
||||
go func(b Boundary) {
|
||||
var e error
|
||||
if b.Protocol == "http" {
|
||||
e = runHTTP(c, b)
|
||||
} else {
|
||||
e = runTCP(c, b)
|
||||
}
|
||||
if e != nil {
|
||||
event("gateway.listener_failed")
|
||||
stop()
|
||||
}
|
||||
}(boundary)
|
||||
}
|
||||
for _, outbound := range c.Outbounds {
|
||||
go func(b Outbound) {
|
||||
if e := runOutbound(c, b); e != nil {
|
||||
event("gateway.outbound_failed")
|
||||
stop()
|
||||
}
|
||||
}(outbound)
|
||||
}
|
||||
|
||||
event("gateway.ready")
|
||||
<-ctx.Done()
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type authority struct {
|
||||
cert *x509.Certificate
|
||||
key *ecdsa.PrivateKey
|
||||
pem []byte
|
||||
}
|
||||
|
||||
func root(t *testing.T) authority {
|
||||
t.Helper()
|
||||
key, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "Synthetic test CA"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
|
||||
der, e := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cert, e := x509.ParseCertificate(der)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return authority{cert, key, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})}
|
||||
}
|
||||
func client(t *testing.T, ca authority, role string, expired bool) *x509.Certificate {
|
||||
t.Helper()
|
||||
key, e := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
identity, _ := url.Parse("spiffe://messenger/" + role)
|
||||
expiry := time.Now().Add(time.Minute)
|
||||
if expired {
|
||||
expiry = time.Now().Add(-time.Minute)
|
||||
}
|
||||
template := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: role}, NotBefore: time.Now().Add(-time.Hour), NotAfter: expiry, URIs: []*url.URL{identity}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, KeyUsage: x509.KeyUsageDigitalSignature}
|
||||
der, e := x509.CreateCertificate(rand.Reader, template, ca.cert, &key.PublicKey, ca.key)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cert, e := x509.ParseCertificate(der)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
return cert
|
||||
}
|
||||
func TestClientAuthorizationUsesCurrentChainIdentityExpiryAndRevocation(t *testing.T) {
|
||||
ca := root(t)
|
||||
foreign := root(t)
|
||||
directory := t.TempDir()
|
||||
conf := Config{Trust: filepath.Join(directory, "ca.pem"), Revoked: filepath.Join(directory, "revoked.json")}
|
||||
if e := os.WriteFile(conf.Trust, ca.pem, 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := os.WriteFile(conf.Revoked, []byte(`{}`), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
boundary := Boundary{Allow: map[string][]string{"spiffe://messenger/lb": {"* /"}}}
|
||||
good := client(t, ca, "lb", false)
|
||||
state := func(leaf *x509.Certificate, issuer authority) tls.ConnectionState {
|
||||
return tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf, issuer.cert}, VerifiedChains: [][]*x509.Certificate{{leaf, issuer.cert}}}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
state tls.ConnectionState
|
||||
allowed bool
|
||||
}{
|
||||
{"authorized", state(good, ca), true},
|
||||
{"no_certificate", tls.ConnectionState{}, false},
|
||||
{"trusted_but_wrong_role", state(client(t, ca, "worker", false), ca), false},
|
||||
{"foreign_authority", state(client(t, foreign, "lb", false), foreign), false},
|
||||
{"expired", state(client(t, ca, "lb", true), ca), false},
|
||||
}
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, e := peer(conf, test.state, boundary)
|
||||
if (e == nil) != test.allowed {
|
||||
t.Fatalf("authorization mismatch: allowed=%v error=%v", test.allowed, e)
|
||||
}
|
||||
})
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]bool{good.SerialNumber.Text(16): true})
|
||||
if e := os.WriteFile(conf.Revoked, raw, 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := peer(conf, state(good, ca), boundary); e == nil {
|
||||
t.Fatal("revoked client accepted on previously authenticated connection")
|
||||
}
|
||||
if e := os.WriteFile(conf.Revoked, []byte(`{}`), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := os.WriteFile(conf.Trust, foreign.pem, 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := peer(conf, state(good, ca), boundary); e == nil {
|
||||
t.Fatal("retired authority accepted from cached verified chain")
|
||||
}
|
||||
if e := os.Remove(conf.Revoked); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := peer(conf, state(good, ca), boundary); e == nil {
|
||||
t.Fatal("missing revocation policy did not fail closed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"role": "grafana",
|
||||
"dns": [
|
||||
"grafana",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:3443",
|
||||
"target": "127.0.0.1:3000",
|
||||
"protocol": "http",
|
||||
"allow": {},
|
||||
"public": true
|
||||
}
|
||||
],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:13909",
|
||||
"target": "https://prometheus:9900",
|
||||
"server_identity": "spiffe://messenger/prometheus"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:13310",
|
||||
"target": "https://loki:3310",
|
||||
"server_identity": "spiffe://messenger/loki"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:13320",
|
||||
"target": "https://tempo:3320",
|
||||
"server_identity": "spiffe://messenger/tempo"
|
||||
}
|
||||
],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"role": "lb",
|
||||
"dns": [
|
||||
"lb",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:18404",
|
||||
"target": "127.0.0.1:8404",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/prometheus": [
|
||||
"GET /metrics"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /metrics"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"role": "loki",
|
||||
"dns": [
|
||||
"loki"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:3310",
|
||||
"target": "127.0.0.1:3100",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/grafana": [
|
||||
"GET /",
|
||||
"POST /api/"
|
||||
],
|
||||
"spiffe://messenger/prometheus": [
|
||||
"GET /metrics"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /"
|
||||
],
|
||||
"spiffe://messenger/otel": [
|
||||
"POST /otlp/"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"role": "migrate",
|
||||
"dns": [
|
||||
"migrate"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "observer",
|
||||
"dns": [
|
||||
"observer"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:14318",
|
||||
"target": "https://otel:4438",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
{
|
||||
"role": "otel",
|
||||
"dns": [
|
||||
"otel"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:4438",
|
||||
"target": "127.0.0.1:4318",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/api": [
|
||||
"POST /v1/"
|
||||
],
|
||||
"spiffe://messenger/worker": [
|
||||
"POST /v1/"
|
||||
],
|
||||
"spiffe://messenger/relay": [
|
||||
"POST /v1/"
|
||||
],
|
||||
"spiffe://messenger/observer": [
|
||||
"POST /v1/"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"POST /v1/"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
},
|
||||
{
|
||||
"listen": "0.0.0.0:18889",
|
||||
"target": "127.0.0.1:8889",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/prometheus": [
|
||||
"GET /metrics"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /metrics"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
},
|
||||
{
|
||||
"listen": "0.0.0.0:18888",
|
||||
"target": "127.0.0.1:8888",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/prometheus": [
|
||||
"GET /metrics"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /metrics"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
},
|
||||
{
|
||||
"listen": "0.0.0.0:13333",
|
||||
"target": "127.0.0.1:13133",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:13310",
|
||||
"target": "https://loki:3310",
|
||||
"server_identity": "spiffe://messenger/loki"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:14319",
|
||||
"target": "https://tempo:4438",
|
||||
"server_identity": "spiffe://messenger/tempo"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"role": "postgres",
|
||||
"dns": [
|
||||
"postgres"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:5443",
|
||||
"target": "127.0.0.1:5432",
|
||||
"protocol": "postgres",
|
||||
"allow": {
|
||||
"spiffe://messenger/api": [],
|
||||
"spiffe://messenger/worker": [],
|
||||
"spiffe://messenger/relay": [],
|
||||
"spiffe://messenger/observer": [],
|
||||
"spiffe://messenger/reconciler": [],
|
||||
"spiffe://messenger/migrate": [],
|
||||
"spiffe://messenger/operator": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"role": "prometheus",
|
||||
"dns": [
|
||||
"prometheus"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:9900",
|
||||
"target": "127.0.0.1:9090",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/grafana": [
|
||||
"GET /api/",
|
||||
"POST /api/v1/query"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:18889",
|
||||
"target": "https://otel:18889",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:18888",
|
||||
"target": "https://otel:18888",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:18404",
|
||||
"target": "https://lb:18404",
|
||||
"server_identity": "spiffe://messenger/lb"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:13200",
|
||||
"target": "https://tempo:3320",
|
||||
"server_identity": "spiffe://messenger/tempo"
|
||||
},
|
||||
{
|
||||
"listen": "127.0.0.1:13100",
|
||||
"target": "https://loki:3310",
|
||||
"server_identity": "spiffe://messenger/loki"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"role": "reconciler",
|
||||
"dns": [
|
||||
"reconciler"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "relay",
|
||||
"dns": [
|
||||
"relay"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:14318",
|
||||
"target": "https://otel:4438",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"role": "s3",
|
||||
"dns": [
|
||||
"s3",
|
||||
"localhost"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:9443",
|
||||
"target": "127.0.0.1:9000",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/api": [
|
||||
"* /"
|
||||
],
|
||||
"spiffe://messenger/worker": [
|
||||
"* /"
|
||||
],
|
||||
"spiffe://messenger/reconciler": [
|
||||
"* /"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"* /"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": [],
|
||||
"ip": [
|
||||
"127.0.0.1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"role": "tempo",
|
||||
"dns": [
|
||||
"tempo"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [
|
||||
{
|
||||
"listen": "0.0.0.0:3320",
|
||||
"target": "127.0.0.1:3200",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/grafana": [
|
||||
"GET /",
|
||||
"POST /api/"
|
||||
],
|
||||
"spiffe://messenger/prometheus": [
|
||||
"GET /metrics"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"GET /"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
},
|
||||
{
|
||||
"listen": "0.0.0.0:4438",
|
||||
"target": "127.0.0.1:4318",
|
||||
"protocol": "http",
|
||||
"allow": {
|
||||
"spiffe://messenger/otel": [
|
||||
"POST /v1/traces"
|
||||
],
|
||||
"spiffe://messenger/operator": [
|
||||
"POST /v1/traces"
|
||||
]
|
||||
},
|
||||
"public": false
|
||||
}
|
||||
],
|
||||
"outbounds": []
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"role": "worker",
|
||||
"dns": [
|
||||
"worker"
|
||||
],
|
||||
"bao": "https://openbao:8200",
|
||||
"trust": "/trust/ca.pem",
|
||||
"revoked": "/trust/revoked.json",
|
||||
"identity": "/run/identity",
|
||||
"ttl": "120s",
|
||||
"serve_only": true,
|
||||
"servers": [],
|
||||
"outbounds": [
|
||||
{
|
||||
"listen": "127.0.0.1:14318",
|
||||
"target": "https://otel:4438",
|
||||
"server_identity": "spiffe://messenger/otel"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX members_user_chat_idx ON members(user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL,
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32),
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY(user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at);
|
||||
@@ -0,0 +1,32 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD UNIQUE(chat_id,id);
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL CHECK(state IN ('pending','ready','failed')),
|
||||
error_code text,
|
||||
uploader_instance text NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(chat_id,id),
|
||||
FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position integer NOT NULL CHECK(position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY(message_id,attachment_id),
|
||||
UNIQUE(message_id,position),
|
||||
FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id),
|
||||
FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id)
|
||||
);
|
||||
@@ -0,0 +1,54 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1);
|
||||
ALTER TABLE attachments ADD COLUMN published_version integer;
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL,
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds'
|
||||
);
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL,
|
||||
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')),
|
||||
attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1,
|
||||
fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz,
|
||||
next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_due_idx ON jobs(state,next_at);
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz
|
||||
);
|
||||
CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL;
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')),
|
||||
object_key text UNIQUE NOT NULL, content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL, sha256 text NOT NULL,
|
||||
width integer, height integer,
|
||||
PRIMARY KEY(attachment_id,pipeline_version,name)
|
||||
);
|
||||
CREATE TABLE effect_publications (
|
||||
attachment_id uuid NOT NULL, pipeline_version integer NOT NULL,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL,
|
||||
committed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY(attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE TABLE pipeline_audit (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL,
|
||||
operator text NOT NULL, reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint.
|
||||
CREATE TABLE pipeline_faults (
|
||||
id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL,
|
||||
action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')),
|
||||
remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz
|
||||
);
|
||||
CREATE TABLE pipeline_events (
|
||||
id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,10 @@
|
||||
ALTER TABLE jobs ADD COLUMN traceparent text NOT NULL DEFAULT '';
|
||||
ALTER TABLE jobs ADD COLUMN tracestate text NOT NULL DEFAULT '';
|
||||
CREATE TABLE job_slo_observations (
|
||||
job_id uuid PRIMARY KEY REFERENCES jobs(id),
|
||||
outcome text NOT NULL CHECK(outcome IN ('good','bad')),
|
||||
decided_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
histogram_epoch uuid
|
||||
);
|
||||
CREATE INDEX job_slo_epoch_idx ON job_slo_observations(histogram_epoch);
|
||||
CREATE INDEX effect_publications_time_idx ON effect_publications(committed_at,job_id);
|
||||
@@ -0,0 +1,32 @@
|
||||
-- Only the owner can touch buckets. Callers get a fixed, serialized function.
|
||||
CREATE TABLE login_buckets (
|
||||
bucket text PRIMARY KEY, tokens double precision NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX login_buckets_expiry_idx ON login_buckets(updated_at);
|
||||
CREATE FUNCTION login_wait(account text) RETURNS integer
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = public, pg_temp AS $$
|
||||
DECLARE
|
||||
moment timestamptz := clock_timestamp();
|
||||
global_tokens double precision; account_tokens double precision;
|
||||
BEGIN
|
||||
IF account !~ '^[a-f0-9]{64}$' THEN RAISE EXCEPTION 'Invalid limiter key'; END IF;
|
||||
INSERT INTO login_buckets VALUES ('global',8,moment) ON CONFLICT DO NOTHING;
|
||||
SELECT least(8.0,tokens+extract(epoch FROM moment-updated_at)*2.0)
|
||||
INTO global_tokens FROM login_buckets WHERE bucket='global' FOR UPDATE;
|
||||
-- Reject before creating arbitrary account buckets: a name flood cannot
|
||||
-- grow state faster than the two admitted attempts per second.
|
||||
IF global_tokens<1 THEN RETURN ceil((1-global_tokens)/2.0)::integer; END IF;
|
||||
INSERT INTO login_buckets VALUES ('account:'||account,8,moment) ON CONFLICT DO NOTHING;
|
||||
SELECT least(8.0,tokens+extract(epoch FROM moment-updated_at)*(8.0/60.0))
|
||||
INTO account_tokens FROM login_buckets WHERE bucket='account:'||account FOR UPDATE;
|
||||
IF account_tokens<1 THEN RETURN ceil((1-account_tokens)/(8.0/60.0))::integer; END IF;
|
||||
UPDATE login_buckets SET tokens=global_tokens-1,updated_at=moment WHERE bucket='global';
|
||||
UPDATE login_buckets SET tokens=account_tokens-1,updated_at=moment WHERE bucket='account:'||account;
|
||||
DELETE FROM login_buckets WHERE bucket IN (
|
||||
SELECT bucket FROM login_buckets WHERE updated_at<moment-interval '10 minutes'
|
||||
AND bucket<>'global' ORDER BY updated_at LIMIT 128
|
||||
);
|
||||
RETURN 0;
|
||||
END $$;
|
||||
REVOKE ALL ON FUNCTION login_wait(text) FROM PUBLIC;
|
||||
@@ -0,0 +1,60 @@
|
||||
[project]
|
||||
name = "mtusi-messenger-lab7"
|
||||
version = "1.0.0"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"fastapi==0.143.3",
|
||||
"pydantic==2.14.1",
|
||||
"uvicorn==0.54.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"argon2-cffi>=25,<26",
|
||||
"redis==8.1.0",
|
||||
"minio==7.2.20",
|
||||
"pillow==13.0.0",
|
||||
"opentelemetry-api==1.45.2",
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"psutil==7.1.3",
|
||||
"cryptography==50.0.2",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8,<10",
|
||||
"httpx>=0.28,<1",
|
||||
"ruff>=0.14,<1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"psutil>=7,<8",
|
||||
"matplotlib>=3.10,<4",
|
||||
"pyyaml==6.0.3",
|
||||
"pip-audit==2.10.1",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 79
|
||||
target-version = "py314"
|
||||
include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
|
||||
# SQL и генерируемые Markdown-таблицы в инструментах не разрываем.
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"research/*.py" = ["E501"]
|
||||
"scripts/*.py" = ["E501"]
|
||||
|
||||
# Keep SQL statements together; the formatter controls executable statements.
|
||||
"src/attachments.py" = ["E501"]
|
||||
"src/pipeline.py" = ["E501"]
|
||||
"src/worker.py" = ["E501"]
|
||||
"src/operator.py" = ["E501"]
|
||||
"src/reconcile.py" = ["E501"]
|
||||
"src/observer.py" = ["E501"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra"
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Summarize measured fault invariants and child resource samples."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def effects(case):
|
||||
for key in ["effect", "v1", "v2"]:
|
||||
if key in case:
|
||||
yield key, case[key]
|
||||
for i, item in enumerate(case.get("effects", [])):
|
||||
yield str(i + 1), item
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("input", type=Path)
|
||||
p.add_argument("--output", required=True, type=Path)
|
||||
a = p.parse_args()
|
||||
d = json.loads(a.input.read_text())
|
||||
assert d["complete"] and d.get("scope", "all") == "all"
|
||||
rows = []
|
||||
for case in d["cases"]:
|
||||
for role, e in effects(case):
|
||||
j = e["job"]
|
||||
rows.append(
|
||||
dict(
|
||||
case=case["case"],
|
||||
role=role,
|
||||
job_id=j["id"],
|
||||
version=j["pipeline_version"],
|
||||
attempt=j["attempt"],
|
||||
cycle=j["cycle"],
|
||||
fence=j["fence"],
|
||||
state=j["state"],
|
||||
publications=e["publications"],
|
||||
variant_rows=len(e["variants"]),
|
||||
physical_keys=len(e["physical_keys"]),
|
||||
)
|
||||
)
|
||||
a.output.mkdir(parents=True, exist_ok=True)
|
||||
with (a.output / "effects.csv").open("w") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
ready = [r for r in rows if r["state"] == "ready"]
|
||||
assert all(
|
||||
r["publications"] == 1
|
||||
and r["variant_rows"] == 4
|
||||
and r["physical_keys"] == 4
|
||||
for r in ready
|
||||
)
|
||||
stale = [r for r in rows if r["state"] == "obsolete"]
|
||||
assert all(r["publications"] == 0 for r in stale)
|
||||
job_ids = {r["job_id"] for r in rows}
|
||||
samples = [
|
||||
x["detail"]
|
||||
for x in d["worker_metrics"]
|
||||
if str(x["job_id"]) in job_ids
|
||||
and x["detail"].get("result") == "ready"
|
||||
and "rss_peak_mib" in x["detail"]
|
||||
]
|
||||
stats = {
|
||||
"source_commit": d["source_commit"],
|
||||
"cases": len(d["cases"]),
|
||||
"observed_ready_effects": len(ready),
|
||||
"observed_obsolete_effects": len(stale),
|
||||
"all_ready_have_one_publication_and_four_variants": True,
|
||||
"all_obsolete_have_zero_publications": True,
|
||||
"successful_child_samples": len(samples),
|
||||
"child_elapsed_range_seconds": [
|
||||
min(x["seconds"] for x in samples),
|
||||
max(x["seconds"] for x in samples),
|
||||
],
|
||||
"child_cpu_range_seconds": [
|
||||
min(x["cpu_seconds"] for x in samples),
|
||||
max(x["cpu_seconds"] for x in samples),
|
||||
],
|
||||
"child_peak_rss_range_mib": [
|
||||
min(x["rss_peak_mib"] for x in samples),
|
||||
max(x["rss_peak_mib"] for x in samples),
|
||||
],
|
||||
"scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned",
|
||||
}
|
||||
(a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained")
|
||||
axes[0].bar(
|
||||
range(1, len(rows) + 1),
|
||||
[r["publications"] for r in rows],
|
||||
color=[
|
||||
"#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows
|
||||
],
|
||||
)
|
||||
axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1)
|
||||
axes[0].set(
|
||||
xlabel="Effect observation (CSV row)",
|
||||
ylabel="Committed publications",
|
||||
title="One publication per ready version; stale v1 = 0",
|
||||
yticks=[0, 1],
|
||||
)
|
||||
axes[1].scatter(
|
||||
[x["seconds"] for x in samples],
|
||||
[x["rss_peak_mib"] for x in samples],
|
||||
color="#246b8e",
|
||||
s=35,
|
||||
alpha=0.8,
|
||||
)
|
||||
axes[1].set(
|
||||
xlabel="Child elapsed, seconds",
|
||||
ylabel="Child peak RSS, MiB",
|
||||
title=f"{len(samples)} completed child samples (synthetic320×200)",
|
||||
)
|
||||
for ax in axes:
|
||||
ax.grid(axis="y", alpha=0.2)
|
||||
fig.savefig(a.output / "pipeline-results.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print(json.dumps(stats, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Derive tables and a figure from completed observations, never simulated data."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def quantile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
return (
|
||||
ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
|
||||
if ordered
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
report = json.loads(args.input.read_text())
|
||||
assert report["complete"] and report["scope"] == "full30m"
|
||||
assert report["elapsed_seconds"] >= 1800
|
||||
with (args.input.parent / report["requests_csv"]).open() as stream:
|
||||
requests = list(csv.DictReader(stream))
|
||||
samples = report["samples"]
|
||||
origin = samples[0]["at"]
|
||||
summaries = []
|
||||
for phase in report["phases"]:
|
||||
rows = [r for r in requests if r["phase"] == phase["name"]]
|
||||
successful = [
|
||||
float(r["seconds"]) for r in rows if r["status"] == "201"
|
||||
]
|
||||
elapsed = phase["end"] - phase["start"]
|
||||
summaries.append(
|
||||
{
|
||||
"phase": phase["name"],
|
||||
"seconds": round(elapsed, 3),
|
||||
"started_requests": len(rows),
|
||||
"statuses": dict(Counter(r["status"] for r in rows)),
|
||||
"successful_rps": round(len(successful) / elapsed, 3),
|
||||
"success_p50_ms": round(quantile(successful, 0.5) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"success_p95_ms": round(quantile(successful, 0.95) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"success_p99_ms": round(quantile(successful, 0.99) * 1000, 3)
|
||||
if successful
|
||||
else None,
|
||||
"critical_firing_names": sorted(
|
||||
n
|
||||
for n in phase["firing"]
|
||||
if n
|
||||
in {
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
}
|
||||
),
|
||||
"warning_firing_names": sorted(
|
||||
n
|
||||
for n in phase["firing"]
|
||||
if n
|
||||
not in {
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
intervals = {}
|
||||
for name in sorted({a["name"] for s in samples for a in s["alerts"]}):
|
||||
segments, start = [], None
|
||||
for sample in samples:
|
||||
active = any(
|
||||
a["name"] == name and a["state"] == "firing"
|
||||
for a in sample["alerts"]
|
||||
)
|
||||
if active and start is None:
|
||||
start = sample["at"]
|
||||
elif not active and start is not None:
|
||||
segments.append([start, sample["at"]])
|
||||
start = None
|
||||
if start is not None:
|
||||
segments.append([start, samples[-1]["at"]])
|
||||
intervals[name] = segments
|
||||
gap = [
|
||||
b["at"] - a["at"] for a, b in zip(samples, samples[1:], strict=False)
|
||||
]
|
||||
memory = [
|
||||
s["signals"]["collector_memory"] / 1024**2
|
||||
for s in samples
|
||||
if s["signals"]["collector_memory"] is not None
|
||||
]
|
||||
summary = {
|
||||
"source_commit": report["source_commit"],
|
||||
"runtime_versions": report["runtime_versions"],
|
||||
"elapsed_seconds": report["elapsed_seconds"],
|
||||
"sample_count": len(samples),
|
||||
"sample_gap_seconds": {
|
||||
"min": min(gap),
|
||||
"median": quantile(gap, 0.5),
|
||||
"max": max(gap),
|
||||
},
|
||||
"client_quantile_method": "nearest rank of successful end-to-end writes; not histogram interpolation",
|
||||
"phases": summaries,
|
||||
"incidents": report["incidents"],
|
||||
"firing_intervals_unix_seconds": intervals,
|
||||
"short_spikes": report["short_spikes"],
|
||||
"collector_rss_mib": {"min": min(memory), "max": max(memory)},
|
||||
"collector_max_queued_packets_sum": max(
|
||||
s["signals"]["collector_queue"] or 0 for s in samples
|
||||
),
|
||||
"collector_queue_capacity_sum": max(
|
||||
s["signals"]["collector_queue_capacity"] or 0 for s in samples
|
||||
),
|
||||
"backend_outage": report["backend_outage"],
|
||||
"preservation": report["preservation"],
|
||||
"final_30m_sli": report["final_30m_sli"],
|
||||
}
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
(args.output / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
with (args.output / "phases.csv").open("w") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(summaries[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
x = [(s["at"] - origin) / 60 for s in samples]
|
||||
fig, axes = plt.subplots(
|
||||
4, 1, figsize=(12, 12), sharex=True, layout="constrained"
|
||||
)
|
||||
for axis in axes:
|
||||
axis.grid(alpha=0.2)
|
||||
for phase in report["phases"]:
|
||||
if (
|
||||
phase["name"].endswith("_sustained")
|
||||
or phase["name"] == "backend_unavailable"
|
||||
):
|
||||
color = (
|
||||
"#cf704e"
|
||||
if phase["name"].startswith("api")
|
||||
else "#8b79b5"
|
||||
if phase["name"].startswith("ready")
|
||||
else "#85919d"
|
||||
)
|
||||
axis.axvspan(
|
||||
(phase["start"] - origin) / 60,
|
||||
(phase["end"] - origin) / 60,
|
||||
color=color,
|
||||
alpha=0.15,
|
||||
)
|
||||
axes[0].plot(
|
||||
x,
|
||||
[(s["signals"]["api_error_ratio_1m"] or 0) * 100 for s in samples],
|
||||
color="#b04a32",
|
||||
label="API 5xx ratio, rolling 1 min",
|
||||
)
|
||||
axes[0].axhline(1, color="black", ls=":", label="1% error budget")
|
||||
axes[0].set_ylabel("5xx, %")
|
||||
axes[0].legend(loc="upper right")
|
||||
axes[0].set_title(
|
||||
"Actual 30-minute SLO experiment: DB delay, stopped worker, unavailable telemetry stores"
|
||||
)
|
||||
axes[1].plot(
|
||||
x,
|
||||
[s["signals"]["queue_depth"] for s in samples],
|
||||
color="#386a9b",
|
||||
label="Queued jobs",
|
||||
)
|
||||
axes[1].set_ylabel("Jobs")
|
||||
other = axes[1].twinx()
|
||||
other.plot(
|
||||
x,
|
||||
[s["signals"]["oldest_job_age"] for s in samples],
|
||||
color="#8b579d",
|
||||
label="Oldest unfinished job",
|
||||
)
|
||||
other.axhline(20, ls=":", color="#8b579d")
|
||||
other.set_ylabel("Age, seconds")
|
||||
axes[1].legend(loc="upper left")
|
||||
other.legend(loc="upper right")
|
||||
alert_names = [
|
||||
"ApiFastBurn",
|
||||
"ApiSlowBurn",
|
||||
"ReadyFastBurn",
|
||||
"ReadySlowBurn",
|
||||
]
|
||||
for index, name in enumerate(alert_names):
|
||||
axes[2].broken_barh(
|
||||
[
|
||||
((start - origin) / 60, (end - start) / 60)
|
||||
for start, end in intervals.get(name, [])
|
||||
],
|
||||
(index - 0.3, 0.6),
|
||||
facecolors="#bd523c",
|
||||
)
|
||||
axes[2].set_yticks(range(4), alert_names)
|
||||
axes[2].set_ylim(-0.6, 3.6)
|
||||
axes[2].set_ylabel("Critical alert firing")
|
||||
axes[3].plot(
|
||||
x,
|
||||
[
|
||||
s["signals"]["collector_memory"] / 1024**2
|
||||
if s["signals"]["collector_memory"] is not None
|
||||
else None
|
||||
for s in samples
|
||||
],
|
||||
color="#347465",
|
||||
label="Collector RSS",
|
||||
)
|
||||
axes[3].axhline(
|
||||
report["container_limits"]["otel"]["mem_limit"] / 1024**2,
|
||||
ls=":",
|
||||
color="black",
|
||||
label="Container memory limit",
|
||||
)
|
||||
axes[3].set_ylabel("MiB")
|
||||
axes[3].set_xlabel("Minutes from first observation")
|
||||
other = axes[3].twinx()
|
||||
other.plot(
|
||||
x,
|
||||
[s["signals"]["collector_queue"] for s in samples],
|
||||
color="#9c6e2e",
|
||||
label="Queued telemetry packets, sum",
|
||||
)
|
||||
other.set_ylabel("Packets")
|
||||
axes[3].legend(loc="upper left")
|
||||
other.legend(loc="upper right")
|
||||
fig.savefig(args.output / "slo-timeline.png", dpi=150)
|
||||
plt.close(fig)
|
||||
print("PASS: measured phase statistics, alert intervals and timeline")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,351 @@
|
||||
"""Explicit operator init/unseal/configure; no unseal material enters containers."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from cryptography import x509
|
||||
|
||||
from scripts.bootstrap_security import (
|
||||
OPERATOR,
|
||||
ROLES,
|
||||
ROOT,
|
||||
TRUST,
|
||||
ca,
|
||||
public,
|
||||
sign_intermediate,
|
||||
write_private,
|
||||
)
|
||||
|
||||
|
||||
class Operator:
|
||||
def __init__(self):
|
||||
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
context.load_cert_chain(
|
||||
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
|
||||
)
|
||||
self.http = httpx.Client(
|
||||
base_url="https://localhost:"
|
||||
+ os.getenv("BAO_PORT", "8207")
|
||||
+ "/v1/",
|
||||
verify=context,
|
||||
trust_env=False,
|
||||
timeout=10,
|
||||
)
|
||||
self.token = None
|
||||
|
||||
def close(self):
|
||||
self.http.close()
|
||||
|
||||
def call(self, method, path, payload=None, *, token=None):
|
||||
response = self.http.request(
|
||||
method,
|
||||
path,
|
||||
json=payload,
|
||||
headers={"X-Vault-Token": token} if token else {},
|
||||
)
|
||||
if response.status_code >= 300:
|
||||
raise RuntimeError(
|
||||
"Operator request denied/unavailable: "
|
||||
+ path
|
||||
+ " status "
|
||||
+ str(response.status_code)
|
||||
)
|
||||
return response.json() if response.content else {}
|
||||
|
||||
def init(self):
|
||||
if self.call("GET", "sys/init")["initialized"]:
|
||||
return
|
||||
result = self.call(
|
||||
"POST", "sys/init", {"secret_shares": 3, "secret_threshold": 2}
|
||||
)
|
||||
for index, key in enumerate(result["keys_base64"]):
|
||||
write_private(
|
||||
OPERATOR / ("unseal-" + str(index + 1) + ".json"),
|
||||
json.dumps({"key": key}) + "\n",
|
||||
)
|
||||
write_private(OPERATOR / "bootstrap-root-token", result["root_token"])
|
||||
|
||||
def unseal(self):
|
||||
state = self.call("GET", "sys/seal-status")
|
||||
if not state["sealed"]:
|
||||
return
|
||||
for index in [1, 2]:
|
||||
share = json.loads(
|
||||
(OPERATOR / ("unseal-" + str(index) + ".json")).read_text()
|
||||
)
|
||||
state = self.call("POST", "sys/unseal", {"key": share["key"]})
|
||||
if not state["sealed"]:
|
||||
return
|
||||
raise RuntimeError("Unseal threshold not reached")
|
||||
|
||||
def authenticate(self):
|
||||
cached = OPERATOR / "operator-token.json"
|
||||
if cached.exists():
|
||||
data = json.loads(cached.read_text())
|
||||
if data["expires_at"] > time.time() + 30:
|
||||
self.token = data["token"]
|
||||
return self.token
|
||||
result = self.call("POST", "auth/cert/login", {})["auth"]
|
||||
self.token = result["client_token"]
|
||||
write_private(
|
||||
cached,
|
||||
json.dumps(
|
||||
{
|
||||
"token": self.token,
|
||||
"expires_at": time.time() + result["lease_duration"],
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
return self.token
|
||||
|
||||
def authorized(self, method, path, payload=None):
|
||||
return self.call(
|
||||
method, path, payload, token=self.token or self.authenticate()
|
||||
)
|
||||
|
||||
def configure(self):
|
||||
marker = OPERATOR / "configured.json"
|
||||
if marker.exists():
|
||||
self.authenticate()
|
||||
self.authorized("GET", "auth/token/lookup-self")
|
||||
return
|
||||
token = (OPERATOR / "bootstrap-root-token").read_text().strip()
|
||||
|
||||
def root(method, path, payload=None):
|
||||
return self.call(method, path, payload, token=token)
|
||||
|
||||
mounts = root("GET", "sys/mounts")
|
||||
mounts = mounts.get("data", mounts)
|
||||
if "secret/" not in mounts:
|
||||
root(
|
||||
"POST",
|
||||
"sys/mounts/secret",
|
||||
{"type": "kv", "options": {"version": "2"}},
|
||||
)
|
||||
if "pki/" not in mounts:
|
||||
root(
|
||||
"POST",
|
||||
"sys/mounts/pki",
|
||||
{"type": "pki", "config": {"max_lease_ttl": "720h"}},
|
||||
)
|
||||
auth = root("GET", "sys/auth")
|
||||
auth = auth.get("data", auth)
|
||||
if "cert/" not in auth:
|
||||
root("POST", "sys/auth/cert", {"type": "cert"})
|
||||
issuer_marker = OPERATOR / "initial-issuer.json"
|
||||
if not issuer_marker.exists():
|
||||
csr = root(
|
||||
"POST",
|
||||
"pki/intermediate/generate/internal",
|
||||
{
|
||||
"common_name": "Messenger service intermediate",
|
||||
"key_type": "ec",
|
||||
"key_bits": 256,
|
||||
},
|
||||
)["data"]["csr"]
|
||||
key, cert = ca()
|
||||
intermediate = sign_intermediate(
|
||||
key, cert, x509.load_pem_x509_csr(csr.encode())
|
||||
)
|
||||
imported = root(
|
||||
"POST",
|
||||
"pki/intermediate/set-signed",
|
||||
{
|
||||
"certificate": (
|
||||
public(intermediate) + public(cert)
|
||||
).decode()
|
||||
},
|
||||
)
|
||||
issuers = imported["data"]["imported_issuers"]
|
||||
selected = None
|
||||
for issuer in issuers:
|
||||
details = root("GET", "pki/issuer/" + issuer)["data"]
|
||||
loaded = x509.load_pem_x509_certificate(
|
||||
details["certificate"].encode()
|
||||
)
|
||||
if loaded.serial_number == intermediate.serial_number:
|
||||
selected = issuer
|
||||
assert selected, "Imported signing intermediate not found"
|
||||
root(
|
||||
"POST",
|
||||
"pki/config/issuers",
|
||||
{"default": selected, "default_follows_latest_issuer": False},
|
||||
)
|
||||
write_private(
|
||||
issuer_marker,
|
||||
json.dumps(
|
||||
{
|
||||
"configured": True,
|
||||
"imported_issuers": imported.get("data", {}).get(
|
||||
"imported_issuers", []
|
||||
),
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
_, cert = ca()
|
||||
material = json.loads((OPERATOR / "service-secrets.json").read_text())
|
||||
for role, cfg in ROLES.items():
|
||||
policy = (
|
||||
ROOT / "infra/openbao/policies" / (role + ".hcl")
|
||||
).read_text()
|
||||
if cfg["kv"]:
|
||||
root(
|
||||
"POST",
|
||||
"secret/data/" + role,
|
||||
{"data": material["services"][role]["values"]},
|
||||
)
|
||||
root(
|
||||
"PUT", "sys/policies/acl/messenger-" + role, {"policy": policy}
|
||||
)
|
||||
root(
|
||||
"POST",
|
||||
"pki/roles/" + role,
|
||||
{
|
||||
"allowed_domains": cfg["dns"],
|
||||
"allow_bare_domains": True,
|
||||
"allow_subdomains": False,
|
||||
"allow_wildcard_certificates": False,
|
||||
"allow_any_name": False,
|
||||
"allow_localhost": "localhost" in cfg["dns"],
|
||||
"allow_ip_sans": bool(cfg.get("ip")),
|
||||
"allowed_uri_sans": ["spiffe://messenger/" + role],
|
||||
"enforce_hostnames": True,
|
||||
"key_type": "ec",
|
||||
"key_bits": 256,
|
||||
"server_flag": True,
|
||||
"client_flag": True,
|
||||
"ttl": "120s",
|
||||
"max_ttl": "120s",
|
||||
"issuer_ref": "default",
|
||||
},
|
||||
)
|
||||
root(
|
||||
"POST",
|
||||
"auth/cert/certs/" + role + "-v1",
|
||||
{
|
||||
"display_name": role,
|
||||
"certificate": public(cert).decode(),
|
||||
"allowed_common_names": [role],
|
||||
"allowed_uri_sans": ["spiffe://messenger/" + role],
|
||||
"token_policies": ["messenger-" + role],
|
||||
"token_ttl": "120s",
|
||||
"token_max_ttl": "600s",
|
||||
"token_no_default_policy": True,
|
||||
},
|
||||
)
|
||||
operator_policy = (
|
||||
ROOT / "infra/openbao/policies/operator.hcl"
|
||||
).read_text()
|
||||
root(
|
||||
"PUT",
|
||||
"sys/policies/acl/messenger-operator",
|
||||
{"policy": operator_policy},
|
||||
)
|
||||
root(
|
||||
"POST",
|
||||
"auth/cert/certs/operator-v1",
|
||||
{
|
||||
"display_name": "operator",
|
||||
"certificate": public(cert).decode(),
|
||||
"allowed_common_names": ["operator"],
|
||||
"allowed_uri_sans": ["spiffe://messenger/operator"],
|
||||
"token_policies": ["messenger-operator"],
|
||||
"token_ttl": "900s",
|
||||
"token_max_ttl": "3600s",
|
||||
"token_no_default_policy": True,
|
||||
},
|
||||
)
|
||||
audit = root("GET", "sys/audit")
|
||||
audit = audit.get("data", audit)
|
||||
if "file/" not in audit:
|
||||
root(
|
||||
"PUT",
|
||||
"sys/audit/file",
|
||||
{
|
||||
"type": "file",
|
||||
"options": {
|
||||
"file_path": "/data/audit/audit.log",
|
||||
"mode": "0600",
|
||||
},
|
||||
},
|
||||
)
|
||||
created = root(
|
||||
"POST",
|
||||
"auth/token/create-orphan",
|
||||
{
|
||||
"policies": ["messenger-operator"],
|
||||
"ttl": "900s",
|
||||
"renewable": True,
|
||||
"no_default_policy": True,
|
||||
},
|
||||
)["auth"]
|
||||
write_private(
|
||||
OPERATOR / "operator-token.json",
|
||||
json.dumps(
|
||||
{
|
||||
"token": created["client_token"],
|
||||
"expires_at": time.time() + created["lease_duration"],
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
root("POST", "auth/token/revoke-self", {})
|
||||
write_private(
|
||||
marker,
|
||||
json.dumps({"configured": True, "root_token_revoked": True})
|
||||
+ "\n",
|
||||
)
|
||||
(OPERATOR / "bootstrap-root-token").unlink()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"action", choices=["init", "unseal", "configure", "status", "seal"]
|
||||
)
|
||||
args = parser.parse_args()
|
||||
operator = Operator()
|
||||
try:
|
||||
if args.action == "init":
|
||||
operator.init()
|
||||
operator.unseal()
|
||||
operator.configure()
|
||||
elif args.action == "unseal":
|
||||
operator.unseal()
|
||||
elif args.action == "configure":
|
||||
operator.configure()
|
||||
elif args.action == "seal":
|
||||
operator.authorized("POST", "sys/seal", {})
|
||||
else:
|
||||
state = operator.call("GET", "sys/seal-status")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
k: state[k]
|
||||
for k in [
|
||||
"initialized",
|
||||
"sealed",
|
||||
"version",
|
||||
"storage_type",
|
||||
]
|
||||
if k in state
|
||||
}
|
||||
)
|
||||
)
|
||||
if args.action != "status":
|
||||
print(
|
||||
"Operator action completed; no tokens or unseal shares printed"
|
||||
)
|
||||
finally:
|
||||
operator.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
stream.write(value + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
pg = directory / "postgres_password"
|
||||
cache = directory / "cache_password"
|
||||
create(pg, secrets.token_urlsafe(32))
|
||||
create(cache, secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "database_url",
|
||||
"postgresql://messenger:"
|
||||
+ pg.read_text().strip()
|
||||
+ "@postgres:5432/messenger",
|
||||
)
|
||||
create(
|
||||
directory / "cache_url",
|
||||
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "valkey.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||
+ cache.read_text().strip(),
|
||||
)
|
||||
create(directory / "s3_root_user", secrets.token_hex(10))
|
||||
create(directory / "s3_root_password", secrets.token_urlsafe(32))
|
||||
create(directory / "s3_access_key", secrets.token_hex(10))
|
||||
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
|
||||
create(directory / "grafana_password", secrets.token_urlsafe(32))
|
||||
create(directory / "broker_password", secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "broker_url",
|
||||
"redis://:"
|
||||
+ (directory / "broker_password").read_text().strip()
|
||||
+ "@broker:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "broker.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
|
||||
+ (directory / "broker_password").read_text().strip(),
|
||||
)
|
||||
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||
print("Local configuration ready; existing files preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Operator-only offline trust/bootstrap generation; never prints key material."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OPERATOR = ROOT / "operator"
|
||||
SECRET = ROOT / "secrets"
|
||||
TRUST = ROOT / "trust"
|
||||
ROLES = {
|
||||
"openbao": {
|
||||
"dns": ["openbao", "localhost"],
|
||||
"ip": ["127.0.0.1"],
|
||||
"uid": 1000,
|
||||
"kv": False,
|
||||
},
|
||||
"api": {"dns": ["api", "localhost"], "uid": 10001, "kv": True},
|
||||
"worker": {"dns": ["worker"], "uid": 10001, "kv": True},
|
||||
"relay": {"dns": ["relay"], "uid": 10001, "kv": True},
|
||||
"observer": {"dns": ["observer"], "uid": 10001, "kv": True},
|
||||
"reconciler": {"dns": ["reconciler"], "uid": 10001, "kv": True},
|
||||
"migrate": {"dns": ["migrate"], "uid": 10001, "kv": True},
|
||||
"lb": {
|
||||
"dns": ["lb", "localhost"],
|
||||
"ip": ["127.0.0.1"],
|
||||
"uid": 99,
|
||||
"kv": False,
|
||||
},
|
||||
"postgres": {"dns": ["postgres"], "uid": 10001, "kv": False},
|
||||
"s3": {
|
||||
"dns": ["s3", "localhost"],
|
||||
"ip": ["127.0.0.1"],
|
||||
"uid": 10001,
|
||||
"kv": False,
|
||||
},
|
||||
"broker": {"dns": ["broker"], "uid": 10001, "kv": False},
|
||||
"cache": {"dns": ["cache"], "uid": 10001, "kv": False},
|
||||
"otel": {"dns": ["otel"], "uid": 10001, "kv": False},
|
||||
"prometheus": {"dns": ["prometheus"], "uid": 65534, "kv": False},
|
||||
"loki": {"dns": ["loki"], "uid": 10001, "kv": False},
|
||||
"tempo": {"dns": ["tempo"], "uid": 10001, "kv": False},
|
||||
"grafana": {
|
||||
"dns": ["grafana", "localhost"],
|
||||
"ip": ["127.0.0.1"],
|
||||
"uid": 472,
|
||||
"kv": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_private(path, content, *, mode=0o600):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
raw = content.encode() if isinstance(content, str) else content
|
||||
temporary = path.with_name(path.name + ".new")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
||||
try:
|
||||
os.fchmod(fd, mode)
|
||||
with os.fdopen(fd, "wb") as stream:
|
||||
stream.write(raw)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def key_bytes(key):
|
||||
return key.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
|
||||
|
||||
def public(cert):
|
||||
return cert.public_bytes(serialization.Encoding.PEM)
|
||||
|
||||
|
||||
def make_root():
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
name = x509.Name(
|
||||
[
|
||||
x509.NameAttribute(
|
||||
NameOID.COMMON_NAME,
|
||||
"Messenger lab root " + secrets.token_hex(4),
|
||||
)
|
||||
]
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - timedelta(minutes=1))
|
||||
.not_valid_after(now + timedelta(days=365))
|
||||
.add_extension(
|
||||
x509.BasicConstraints(ca=True, path_length=1), critical=True
|
||||
)
|
||||
.add_extension(
|
||||
x509.KeyUsage(
|
||||
digital_signature=True,
|
||||
content_commitment=False,
|
||||
key_encipherment=False,
|
||||
data_encipherment=False,
|
||||
key_agreement=False,
|
||||
key_cert_sign=True,
|
||||
crl_sign=True,
|
||||
encipher_only=False,
|
||||
decipher_only=False,
|
||||
),
|
||||
critical=True,
|
||||
)
|
||||
.add_extension(
|
||||
x509.SubjectKeyIdentifier.from_public_key(key.public_key()),
|
||||
critical=False,
|
||||
)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
return key, cert
|
||||
|
||||
|
||||
def sign_leaf(ca_key, ca_cert, role, dns, ips=(), *, seconds=1800):
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
now = datetime.now(UTC)
|
||||
names = [x509.DNSName(name) for name in dns]
|
||||
names += [x509.IPAddress(ipaddress.ip_address(ip)) for ip in ips]
|
||||
names += [x509.UniformResourceIdentifier("spiffe://messenger/" + role)]
|
||||
cert = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(
|
||||
x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, role)])
|
||||
)
|
||||
.issuer_name(ca_cert.subject)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - timedelta(seconds=30))
|
||||
.not_valid_after(now + timedelta(seconds=seconds))
|
||||
.add_extension(
|
||||
x509.BasicConstraints(ca=False, path_length=None), critical=True
|
||||
)
|
||||
.add_extension(x509.SubjectAlternativeName(names), critical=False)
|
||||
.add_extension(
|
||||
x509.ExtendedKeyUsage(
|
||||
[
|
||||
ExtendedKeyUsageOID.CLIENT_AUTH,
|
||||
ExtendedKeyUsageOID.SERVER_AUTH,
|
||||
]
|
||||
),
|
||||
critical=False,
|
||||
)
|
||||
.add_extension(
|
||||
x509.KeyUsage(
|
||||
digital_signature=True,
|
||||
content_commitment=False,
|
||||
key_encipherment=False,
|
||||
data_encipherment=False,
|
||||
key_agreement=False,
|
||||
key_cert_sign=False,
|
||||
crl_sign=False,
|
||||
encipher_only=False,
|
||||
decipher_only=False,
|
||||
),
|
||||
critical=True,
|
||||
)
|
||||
.add_extension(
|
||||
x509.AuthorityKeyIdentifier.from_issuer_public_key(
|
||||
ca_key.public_key()
|
||||
),
|
||||
critical=False,
|
||||
)
|
||||
.sign(ca_key, hashes.SHA256())
|
||||
)
|
||||
return key, cert
|
||||
|
||||
|
||||
def sign_intermediate(ca_key, ca_cert, csr):
|
||||
now = datetime.now(UTC)
|
||||
assert csr.is_signature_valid
|
||||
return (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(csr.subject)
|
||||
.issuer_name(ca_cert.subject)
|
||||
.public_key(csr.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - timedelta(seconds=30))
|
||||
.not_valid_after(now + timedelta(days=30))
|
||||
.add_extension(
|
||||
x509.BasicConstraints(ca=True, path_length=0), critical=True
|
||||
)
|
||||
.add_extension(
|
||||
x509.KeyUsage(
|
||||
digital_signature=True,
|
||||
content_commitment=False,
|
||||
key_encipherment=False,
|
||||
data_encipherment=False,
|
||||
key_agreement=False,
|
||||
key_cert_sign=True,
|
||||
crl_sign=True,
|
||||
encipher_only=False,
|
||||
decipher_only=False,
|
||||
),
|
||||
critical=True,
|
||||
)
|
||||
.add_extension(
|
||||
x509.SubjectKeyIdentifier.from_public_key(csr.public_key()),
|
||||
critical=False,
|
||||
)
|
||||
.add_extension(
|
||||
x509.AuthorityKeyIdentifier.from_issuer_public_key(
|
||||
ca_key.public_key()
|
||||
),
|
||||
critical=False,
|
||||
)
|
||||
.sign(ca_key, hashes.SHA256())
|
||||
)
|
||||
|
||||
|
||||
def ca():
|
||||
return (
|
||||
serialization.load_pem_private_key(
|
||||
(OPERATOR / "ca-key.pem").read_bytes(), password=None
|
||||
),
|
||||
x509.load_pem_x509_certificate((OPERATOR / "ca.pem").read_bytes()),
|
||||
)
|
||||
|
||||
|
||||
def generate():
|
||||
for directory in [OPERATOR, SECRET]:
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
directory.chmod(0o700)
|
||||
TRUST.mkdir(mode=0o755, exist_ok=True)
|
||||
if not (OPERATOR / "ca-key.pem").exists():
|
||||
key, cert = make_root()
|
||||
write_private(OPERATOR / "ca-key.pem", key_bytes(key))
|
||||
write_private(OPERATOR / "ca.pem", public(cert))
|
||||
key, cert = ca()
|
||||
if not (TRUST / "ca.pem").exists():
|
||||
write_private(TRUST / "ca.pem", public(cert), mode=0o644)
|
||||
write_private(TRUST / "revoked.json", "{}\n", mode=0o644)
|
||||
for role, cfg in {
|
||||
**ROLES,
|
||||
"openbao": {"dns": ["openbao", "localhost"], "ip": ["127.0.0.1"]},
|
||||
"operator": {"dns": ["operator"]},
|
||||
}.items():
|
||||
directory = (
|
||||
(OPERATOR / "client")
|
||||
if role == "operator"
|
||||
else SECRET / "bootstrap" / role
|
||||
)
|
||||
renew_operator = (
|
||||
role == "operator"
|
||||
and (directory / "cert.pem").exists()
|
||||
and x509.load_pem_x509_certificate(
|
||||
(directory / "cert.pem").read_bytes()
|
||||
).not_valid_after_utc
|
||||
< datetime.now(UTC) + timedelta(hours=1)
|
||||
)
|
||||
renew_bootstrap = (
|
||||
role != "operator"
|
||||
and (directory / "cert.pem").exists()
|
||||
and x509.load_pem_x509_certificate(
|
||||
(directory / "cert.pem").read_bytes()
|
||||
).not_valid_after_utc
|
||||
< datetime.now(UTC) + timedelta(minutes=5)
|
||||
)
|
||||
wrong_issuer = (
|
||||
directory / "cert.pem"
|
||||
).exists() and x509.load_pem_x509_certificate(
|
||||
(directory / "cert.pem").read_bytes()
|
||||
).issuer != cert.subject
|
||||
# Only this explicit operator command has the offline CA. Agents
|
||||
# cannot refresh a bootstrap credential by themselves after expiry.
|
||||
if (
|
||||
not (directory / "key.pem").exists()
|
||||
or renew_operator
|
||||
or renew_bootstrap
|
||||
or wrong_issuer
|
||||
):
|
||||
leaf_key, leaf = sign_leaf(
|
||||
key,
|
||||
cert,
|
||||
role,
|
||||
cfg["dns"],
|
||||
cfg.get("ip", ()),
|
||||
seconds=86400 if role in {"operator", "openbao"} else 1800,
|
||||
)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
directory.chmod(0o700 if role == "operator" else 0o755)
|
||||
mode = 0o600 if role == "operator" else 0o644
|
||||
write_private(
|
||||
directory / "key.pem", key_bytes(leaf_key), mode=mode
|
||||
)
|
||||
write_private(
|
||||
directory / "cert.pem", public(leaf) + public(cert), mode=mode
|
||||
)
|
||||
write_private(
|
||||
directory / "bundle.pem",
|
||||
public(leaf) + public(cert) + key_bytes(leaf_key),
|
||||
mode=mode,
|
||||
)
|
||||
path = OPERATOR / "service-secrets.json"
|
||||
if path.exists():
|
||||
raw = json.loads(path.read_text())
|
||||
else:
|
||||
raw = {
|
||||
name: secrets.token_urlsafe(32)
|
||||
for name in [
|
||||
"postgres_password",
|
||||
"s3_root_password",
|
||||
"broker_admin",
|
||||
"cache_admin",
|
||||
"cache_api",
|
||||
"grafana_password",
|
||||
]
|
||||
}
|
||||
raw["s3_root_user"] = "operator-" + secrets.token_hex(12)
|
||||
raw["services"] = {}
|
||||
for role in [
|
||||
"api",
|
||||
"worker",
|
||||
"relay",
|
||||
"observer",
|
||||
"reconciler",
|
||||
"migrate",
|
||||
]:
|
||||
pwd = secrets.token_urlsafe(32)
|
||||
values = {
|
||||
"DATABASE_URL": "postgresql://"
|
||||
+ ("messenger" if role == "migrate" else "svc_" + role)
|
||||
+ ":"
|
||||
+ quote(
|
||||
raw["postgres_password"] if role == "migrate" else pwd,
|
||||
safe="",
|
||||
)
|
||||
+ "@postgres:5443/messenger?sslmode=verify-full&gssencmode=disable&sslrootcert=/trust/ca.pem&sslcert=/run/identity/current/cert.pem&sslkey=/run/identity/current/key.pem"
|
||||
}
|
||||
service = {"database_password": pwd, "values": values}
|
||||
if role == "api":
|
||||
values["CACHE_URL"] = (
|
||||
"rediss://api:" + raw["cache_api"] + "@cache:6380/0"
|
||||
)
|
||||
if role in {"relay", "worker"}:
|
||||
service["broker_password"] = secrets.token_urlsafe(32)
|
||||
values["BROKER_URL"] = (
|
||||
"rediss://"
|
||||
+ role
|
||||
+ ":"
|
||||
+ service["broker_password"]
|
||||
+ "@broker:6380/0"
|
||||
)
|
||||
if role in {"api", "worker", "reconciler"}:
|
||||
values.update(
|
||||
S3_ACCESS_KEY=role + "-" + secrets.token_hex(12),
|
||||
S3_SECRET_KEY=secrets.token_urlsafe(32),
|
||||
)
|
||||
raw["services"][role] = service
|
||||
write_private(path, json.dumps(raw, indent=2) + "\n")
|
||||
for name in [
|
||||
"postgres_password",
|
||||
"s3_root_user",
|
||||
"s3_root_password",
|
||||
"grafana_password",
|
||||
]:
|
||||
write_private(SECRET / name, raw[name] + "\n", mode=0o644)
|
||||
for name in ["broker", "cache"]:
|
||||
admin = raw[name + "_admin"]
|
||||
lines = [
|
||||
"bind 127.0.0.1",
|
||||
"port 6379",
|
||||
"protected-mode yes",
|
||||
'save ""',
|
||||
"maxmemory 64mb" if name == "cache" else "maxmemory 192mb",
|
||||
"maxmemory-policy allkeys-lru"
|
||||
if name == "cache"
|
||||
else "maxmemory-policy noeviction",
|
||||
"user default off",
|
||||
"user admin on #"
|
||||
+ hashlib.sha256(admin.encode()).hexdigest()
|
||||
+ " ~* &* +@all",
|
||||
]
|
||||
if name == "cache":
|
||||
lines += [
|
||||
"user api on #"
|
||||
+ hashlib.sha256(raw["cache_api"].encode()).hexdigest()
|
||||
+ " ~user:* +get +set +ping +select +client|setinfo +client|setname"
|
||||
]
|
||||
else:
|
||||
lines += ["appendonly yes", "appendfsync always", "dir /data"]
|
||||
for role, commands in [
|
||||
("relay", "+xadd +waitaof"),
|
||||
(
|
||||
"worker",
|
||||
"+xgroup|create +xreadgroup +xautoclaim +xack +xdel",
|
||||
),
|
||||
]:
|
||||
lines += [
|
||||
"user "
|
||||
+ role
|
||||
+ " on #"
|
||||
+ hashlib.sha256(
|
||||
raw["services"][role]["broker_password"].encode()
|
||||
).hexdigest()
|
||||
+ " ~attachments +ping +select +client|setinfo +client|setname "
|
||||
+ commands
|
||||
]
|
||||
write_private(
|
||||
SECRET / (name + ".conf"), "\n".join(lines) + "\n", mode=0o644
|
||||
)
|
||||
write_private(SECRET / (name + "_admin"), admin + "\n", mode=0o644)
|
||||
|
||||
|
||||
def main():
|
||||
argparse.ArgumentParser(description=__doc__).parse_args()
|
||||
generate()
|
||||
print(
|
||||
"Local operator trust, scoped service credentials and bootstrap certificates prepared; existing secrets retained"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Явные операторские действия только над API текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = command("docker", "compose", "ps", "-q", "api").split()
|
||||
return json.loads(command("docker", "inspect", *ids)) if ids else []
|
||||
|
||||
|
||||
def alias(cid):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=2); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
return command("docker", "exec", cid, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_count(count):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
data = containers()
|
||||
if len(data) == count and all(
|
||||
c["State"].get("Health", {}).get("Status") == "healthy"
|
||||
for c in data
|
||||
):
|
||||
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("Replica readiness")
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
data = containers()
|
||||
if action == "stop":
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [c for c in data if alias(c["Id"]) == target]
|
||||
if len(matches) != 1 or len(data) < 2:
|
||||
raise RuntimeError("Target must identify one running replica")
|
||||
command("docker", "stop", "--time", "10", matches[0]["Id"])
|
||||
print("Selected API stopped")
|
||||
elif action == "rolling":
|
||||
count = len(data)
|
||||
if count < 2:
|
||||
raise RuntimeError("Need at least two replicas")
|
||||
# Сначала добавляется готовая реплика, затем удаляется одна старая.
|
||||
for old in data:
|
||||
command(
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--scale",
|
||||
f"api={count + 1}",
|
||||
"api",
|
||||
)
|
||||
wait_count(count + 1)
|
||||
command("docker", "stop", "--time", "10", old["Id"])
|
||||
command("docker", "rm", old["Id"])
|
||||
wait_count(count)
|
||||
print("Sequential replacement completed")
|
||||
else:
|
||||
raise ValueError("Unknown action")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Operator helpers for explicit certificate repair and working-key rotation."""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import ssl
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.credentials import StaticProvider
|
||||
from minio.error import MinioAdminException, S3Error
|
||||
from minio.minioadmin import MinioAdmin
|
||||
|
||||
from scripts.bao_operator import Operator
|
||||
from scripts.bootstrap_security import OPERATOR, ROLES, TRUST, write_private
|
||||
from scripts.prepare_dependencies import attach_policy
|
||||
from scripts.stand import compose
|
||||
|
||||
INSTALL = """import json,os,pathlib,signal,sys,tempfile
|
||||
data=json.load(sys.stdin)
|
||||
base=pathlib.Path('/run/identity')
|
||||
directory=pathlib.Path(tempfile.mkdtemp(prefix='generation-',dir=base))
|
||||
certificate=data['certificate']+'\\n'+'\\n'.join(data.get('ca_chain',[]))
|
||||
for name,value in [('cert.pem',certificate),('key.pem',data['private_key']),('bundle.pem',certificate+'\\n'+data['private_key'])]:
|
||||
path=directory/name;path.write_text(value);path.chmod(0o600)
|
||||
link=base/'operator-next';link.unlink(missing_ok=True);link.symlink_to(directory.name);link.replace(base/'current')
|
||||
if data.get('reload_lb'):os.kill(1,signal.SIGUSR2)
|
||||
print('Operator installed a scoped short-lived leaf')
|
||||
"""
|
||||
|
||||
|
||||
def install(role, data):
|
||||
payload = {**data, "reload_lb": role == "lb"}
|
||||
args = ["--profile", "maintenance"] if role == "migrate" else []
|
||||
compose(
|
||||
*args,
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"identity-" + role,
|
||||
"-c",
|
||||
INSTALL,
|
||||
input=json.dumps(payload),
|
||||
)
|
||||
|
||||
|
||||
def issue(operator, role):
|
||||
cfg = ROLES[role]
|
||||
request = {
|
||||
"common_name": role,
|
||||
"alt_names": ",".join(cfg["dns"]),
|
||||
"uri_sans": "spiffe://messenger/" + role,
|
||||
"ttl": "120s",
|
||||
}
|
||||
if cfg.get("ip"):
|
||||
request["ip_sans"] = ",".join(cfg["ip"])
|
||||
return operator.authorized("POST", "pki/issue/" + role, request)["data"]
|
||||
|
||||
|
||||
def repair(operator, role):
|
||||
result = issue(operator, role)
|
||||
install(role, result)
|
||||
return {key: result[key] for key in ["serial_number", "expiration"]}
|
||||
|
||||
|
||||
def admin():
|
||||
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
|
||||
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
context.load_cert_chain(
|
||||
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
|
||||
)
|
||||
pool = urllib3.PoolManager(
|
||||
ssl_context=context,
|
||||
timeout=urllib3.Timeout(connect=2, read=8),
|
||||
retries=False,
|
||||
)
|
||||
return (
|
||||
raw,
|
||||
MinioAdmin(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=StaticProvider(
|
||||
raw["s3_root_user"], raw["s3_root_password"]
|
||||
),
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
),
|
||||
pool,
|
||||
)
|
||||
|
||||
|
||||
def rotate_s3(operator):
|
||||
raw, client, pool = admin()
|
||||
try:
|
||||
journal_path = OPERATOR / "s3-rotation.json"
|
||||
resumed = journal_path.exists() and not json.loads(
|
||||
journal_path.read_text()
|
||||
).get("complete")
|
||||
journal = json.loads(journal_path.read_text()) if resumed else {}
|
||||
values = journal.get(
|
||||
"previous_values", raw["services"]["api"]["values"]
|
||||
)
|
||||
previous = values["S3_ACCESS_KEY"]
|
||||
old = Minio(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=StaticProvider(previous, values["S3_SECRET_KEY"]),
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
region="us-east-1",
|
||||
)
|
||||
probe_key = journal.get(
|
||||
"probe_key", "objects/staging/rotation-" + uuid4().hex
|
||||
)
|
||||
current = journal.get("next_values") or {
|
||||
**values,
|
||||
"S3_ACCESS_KEY": "api-" + secrets.token_hex(12),
|
||||
"S3_SECRET_KEY": secrets.token_urlsafe(32),
|
||||
}
|
||||
if not resumed:
|
||||
old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
|
||||
journal = {
|
||||
"previous_values": values,
|
||||
"next_values": current,
|
||||
"probe_key": probe_key,
|
||||
"complete": False,
|
||||
}
|
||||
write_private(journal_path, json.dumps(journal) + "\n")
|
||||
client.user_add(current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"])
|
||||
attach_policy(client, "messenger-api", current["S3_ACCESS_KEY"])
|
||||
operator.authorized("POST", "secret/data/api", {"data": current})
|
||||
# Compare a fingerprint; credential values are not placed in argv/output.
|
||||
import hashlib
|
||||
|
||||
expected = hashlib.sha256(
|
||||
current["S3_ACCESS_KEY"].encode()
|
||||
).hexdigest()
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
observed = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"identity-api",
|
||||
"python",
|
||||
"-c",
|
||||
"import hashlib,json;print(hashlib.sha256(json.load(open('/run/identity/secrets.json'))['values']['S3_ACCESS_KEY'].encode()).hexdigest())",
|
||||
)
|
||||
if observed == expected:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise TimeoutError("New scoped S3 key not observed")
|
||||
raw["services"]["api"]["values"] = current
|
||||
write_private(
|
||||
OPERATOR / "service-secrets.json", json.dumps(raw, indent=2) + "\n"
|
||||
)
|
||||
try:
|
||||
client.user_remove(previous)
|
||||
except MinioAdminException as error:
|
||||
if json.loads(error._body).get("Code") != "XMinioAdminNoSuchUser":
|
||||
raise
|
||||
pool.clear()
|
||||
try:
|
||||
old.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
|
||||
except S3Error as error:
|
||||
assert error.code in {"AccessDenied", "InvalidAccessKeyId"}, (
|
||||
error.code
|
||||
)
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Removed working key accepted on a fresh connection"
|
||||
)
|
||||
fresh = Minio(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=StaticProvider(
|
||||
current["S3_ACCESS_KEY"], current["S3_SECRET_KEY"]
|
||||
),
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
region="us-east-1",
|
||||
)
|
||||
fresh.put_object("messenger", probe_key, io.BytesIO(b"x"), 1)
|
||||
root = Minio(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=StaticProvider(
|
||||
raw["s3_root_user"], raw["s3_root_password"]
|
||||
),
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
region="us-east-1",
|
||||
)
|
||||
root.remove_object("messenger", probe_key)
|
||||
write_private(
|
||||
journal_path, json.dumps({**journal, "complete": True}) + "\n"
|
||||
)
|
||||
return {
|
||||
"new_key_observed": True,
|
||||
"old_account_removed": True,
|
||||
"old_key_fresh_connection_denied": True,
|
||||
"new_key_put_succeeded": True,
|
||||
"image_rebuilt": False,
|
||||
}
|
||||
finally:
|
||||
pool.clear()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=["repair-leaf", "rotate-s3"])
|
||||
parser.add_argument("--role", choices=ROLES, default="lb")
|
||||
args = parser.parse_args()
|
||||
operator = Operator()
|
||||
try:
|
||||
result = (
|
||||
repair(operator, args.role)
|
||||
if args.action == "repair-leaf"
|
||||
else rotate_s3(operator)
|
||||
)
|
||||
print(json.dumps(result))
|
||||
finally:
|
||||
operator.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Read-only queries through an internal probe, without exposing admin ports."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scripts.stand import compose
|
||||
|
||||
ENDPOINTS = {
|
||||
"prometheus": "https://prometheus:9900",
|
||||
"loki": "https://loki:3310",
|
||||
"tempo": "https://tempo:3320",
|
||||
"otel": "https://otel:13333",
|
||||
}
|
||||
|
||||
|
||||
def fetch(service, path, *, params=None, json_result=True):
|
||||
url = ENDPOINTS[service] + path
|
||||
if params:
|
||||
url += "?" + urlencode(params, doseq=True)
|
||||
code = (
|
||||
f"import urllib.request; from src.security_probe import context; service={service!r}; "
|
||||
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); "
|
||||
"print(r.read().decode())"
|
||||
)
|
||||
text = compose(
|
||||
"--profile",
|
||||
"operator",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"operator-probe",
|
||||
"-c",
|
||||
code,
|
||||
)
|
||||
return json.loads(text) if json_result else text
|
||||
|
||||
|
||||
def query(expression, *, at=None):
|
||||
params = {"query": expression}
|
||||
if at is not None:
|
||||
params["time"] = str(at)
|
||||
result = fetch("prometheus", "/api/v1/query", params=params)
|
||||
assert result["status"] == "success"
|
||||
return result["data"]["result"]
|
||||
|
||||
|
||||
def scalar(expression):
|
||||
result = query(expression)
|
||||
return float(result[0]["value"][1]) if result else 0.0
|
||||
|
||||
|
||||
def queries(expressions):
|
||||
urls = {
|
||||
name: ENDPOINTS["prometheus"]
|
||||
+ "/api/v1/query?"
|
||||
+ urlencode({"query": expression})
|
||||
for name, expression in expressions.items()
|
||||
}
|
||||
code = (
|
||||
"import json,urllib.request\nfrom src.security_probe import context\n"
|
||||
f"urls={urls!r}\n"
|
||||
"result={}\n"
|
||||
"for name,url in urls.items():\n"
|
||||
" with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n"
|
||||
"print(json.dumps(result))"
|
||||
)
|
||||
return json.loads(
|
||||
compose(
|
||||
"--profile",
|
||||
"operator",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"operator-probe",
|
||||
"-c",
|
||||
code,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def logs(trace_id, *, start=None):
|
||||
return fetch(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
params={
|
||||
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
|
||||
+ trace_id
|
||||
+ '"',
|
||||
"start": str(int((start or time.time() - 3600) * 1e9)),
|
||||
"end": str(time.time_ns()),
|
||||
"limit": 500,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def normalize_id(value):
|
||||
if not value:
|
||||
return ""
|
||||
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
|
||||
return value
|
||||
return base64.b64decode(value).hex()
|
||||
|
||||
|
||||
def spans(trace_id):
|
||||
result = fetch("tempo", "/api/traces/" + trace_id)
|
||||
flattened = []
|
||||
for batch in result.get("batches", result.get("resourceSpans", [])):
|
||||
resource = {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in batch.get("resource", {}).get("attributes", [])
|
||||
}
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
for item in scope.get("spans", []):
|
||||
flattened.append(
|
||||
{
|
||||
"service": resource.get("service.name"),
|
||||
"version": resource.get("service.version"),
|
||||
"instance": resource.get("service.instance.id"),
|
||||
"name": item["name"],
|
||||
"kind": item.get("kind"),
|
||||
"trace_id": normalize_id(item.get("traceId")),
|
||||
"span_id": normalize_id(item.get("spanId")),
|
||||
"parent_span_id": normalize_id(
|
||||
item.get("parentSpanId")
|
||||
),
|
||||
"duration_seconds": (
|
||||
int(item["endTimeUnixNano"])
|
||||
- int(item["startTimeUnixNano"])
|
||||
)
|
||||
/ 1e9,
|
||||
"status": item.get("status", {}),
|
||||
"attributes": {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in item.get("attributes", [])
|
||||
},
|
||||
}
|
||||
)
|
||||
return flattened
|
||||
|
||||
|
||||
def wait(condition, *, timeout=90, interval=1):
|
||||
end = time.monotonic() + timeout
|
||||
last_error = None
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
except (RuntimeError, ValueError, KeyError) as exc:
|
||||
last_error = type(exc).__name__
|
||||
time.sleep(interval)
|
||||
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Prepare a 60-second operator-only snapshot and run the audited pipeline CLI."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
from scripts.bootstrap_security import OPERATOR, SECRET, write_private
|
||||
from scripts.stand import compose
|
||||
|
||||
|
||||
def prepare():
|
||||
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
|
||||
directory = SECRET / "operator-task"
|
||||
directory.mkdir(mode=0o755, exist_ok=True)
|
||||
for name in ["cert.pem", "key.pem"]:
|
||||
write_private(
|
||||
directory / "current" / name,
|
||||
(OPERATOR / "client" / name).read_bytes(),
|
||||
mode=0o644,
|
||||
)
|
||||
values = {
|
||||
"DATABASE_URL": "postgresql://messenger:"
|
||||
+ quote(raw["postgres_password"], safe="")
|
||||
+ "@postgres:5443/messenger?sslmode=verify-full&gssencmode=disable&sslrootcert=/trust/ca.pem&sslcert=/run/identity/current/cert.pem&sslkey=/run/identity/current/key.pem",
|
||||
"BROKER_URL": "rediss://admin:"
|
||||
+ raw["broker_admin"]
|
||||
+ "@broker:6380/0",
|
||||
"S3_ACCESS_KEY": raw["s3_root_user"],
|
||||
"S3_SECRET_KEY": raw["s3_root_password"],
|
||||
}
|
||||
write_private(
|
||||
directory / "secrets.json",
|
||||
json.dumps({"valid_until": time.time() + 60, "values": values}) + "\n",
|
||||
mode=0o644,
|
||||
)
|
||||
|
||||
|
||||
def run(*args):
|
||||
prepare()
|
||||
return compose(
|
||||
"--profile",
|
||||
"operator",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"operator-task",
|
||||
*args,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
print(run(*sys.argv[1:]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
"""Explicit operator PKI lifecycle; trust is never disabled."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
|
||||
from scripts.bao_operator import Operator
|
||||
from scripts.bootstrap_security import (
|
||||
OPERATOR,
|
||||
ROLES,
|
||||
SECRET,
|
||||
TRUST,
|
||||
ca,
|
||||
key_bytes,
|
||||
make_root,
|
||||
public,
|
||||
sign_intermediate,
|
||||
sign_leaf,
|
||||
write_private,
|
||||
)
|
||||
from scripts.credentials import install, repair
|
||||
from scripts.stand import compose
|
||||
|
||||
REQUEST = (
|
||||
b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
|
||||
|
||||
def fixture_check(role, *, reject=False):
|
||||
expression = f"exchange('api',8443,{role!r},{REQUEST!r})"
|
||||
assertion = (
|
||||
"denied(lambda:" + expression + ")"
|
||||
if reject
|
||||
else "b' 200 ' in " + expression
|
||||
)
|
||||
code = (
|
||||
"from src.security_probe import exchange,denied;assert "
|
||||
+ assertion
|
||||
+ ";print('Fresh TLS fixture check passed')"
|
||||
)
|
||||
compile(code, "fixture-check", "exec")
|
||||
return compose(
|
||||
"--profile",
|
||||
"operator",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"operator-probe",
|
||||
"-c",
|
||||
code,
|
||||
)
|
||||
|
||||
|
||||
META = """import json,pathlib
|
||||
from cryptography import x509
|
||||
result={}
|
||||
for directory in pathlib.Path('/identities').iterdir():
|
||||
try:
|
||||
cert=x509.load_pem_x509_certificate((directory/'current/cert.pem').read_bytes())
|
||||
result[directory.name]={'serial':format(cert.serial_number,'x'),'issuer':cert.issuer.rfc4514_string(),'expires_at':cert.not_valid_after_utc.timestamp()}
|
||||
except FileNotFoundError:pass
|
||||
print(json.dumps(result))
|
||||
"""
|
||||
|
||||
|
||||
def metadata():
|
||||
return json.loads(
|
||||
compose(
|
||||
"--profile",
|
||||
"operator",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"operator-probe",
|
||||
"-c",
|
||||
META,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def reload_lb():
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"identity-lb",
|
||||
"python",
|
||||
"-c",
|
||||
"import os,signal;os.kill(1,signal.SIGUSR2)",
|
||||
)
|
||||
|
||||
|
||||
def preserve_lb():
|
||||
data = json.loads(
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"identity-lb",
|
||||
"python",
|
||||
"-c",
|
||||
"import json,pathlib;p=pathlib.Path('/run/identity/current');print(json.dumps({name:(p/name).read_text() for name in ['cert.pem','key.pem']}))",
|
||||
)
|
||||
)
|
||||
directory = SECRET / "probe/revoked-lb"
|
||||
for name, value in data.items():
|
||||
write_private(directory / name, value, mode=0o644)
|
||||
leaf = x509.load_pem_x509_certificate(data["cert.pem"].encode())
|
||||
return {
|
||||
"serial": format(leaf.serial_number, "x"),
|
||||
"expires_at": leaf.not_valid_after_utc.timestamp(),
|
||||
}
|
||||
|
||||
|
||||
def revoke(operator, leaf):
|
||||
assert leaf["expires_at"] > time.time() + 5, (
|
||||
"Revocation must be tested before natural expiry"
|
||||
)
|
||||
serial = leaf["serial"]
|
||||
serial = ("0" + serial) if len(serial) % 2 else serial
|
||||
colon = ":".join(serial[i : i + 2] for i in range(0, len(serial), 2))
|
||||
operator.authorized("POST", "pki/revoke", {"serial_number": colon})
|
||||
response = operator.http.get(
|
||||
"pki/crl/pem",
|
||||
headers={"X-Vault-Token": operator.token or operator.authenticate()},
|
||||
)
|
||||
response.raise_for_status()
|
||||
operator.authorized(
|
||||
"PUT", "auth/cert/crls/service-issuer", {"crl": response.text}
|
||||
)
|
||||
revoked = json.loads((TRUST / "revoked.json").read_text())
|
||||
revoked[leaf["serial"]] = True
|
||||
write_private(
|
||||
TRUST / "revoked.json", json.dumps(revoked) + "\n", mode=0o644
|
||||
)
|
||||
fixture_check("revoked-lb", reject=True)
|
||||
return {
|
||||
"revoked_serial": leaf["serial"],
|
||||
"still_unexpired_at_check": leaf["expires_at"] > time.time(),
|
||||
"new_tls_connection_denied": True,
|
||||
"bao_crl_updated": True,
|
||||
}
|
||||
|
||||
|
||||
def reload_bao():
|
||||
compose("restart", "openbao")
|
||||
last = None
|
||||
for _ in range(60):
|
||||
operator = Operator()
|
||||
try:
|
||||
operator.unseal()
|
||||
operator.authenticate()
|
||||
return
|
||||
except Exception as error:
|
||||
last = type(error).__name__
|
||||
finally:
|
||||
operator.close()
|
||||
time.sleep(1)
|
||||
raise TimeoutError("Verified Bao restart/unseal unavailable: " + str(last))
|
||||
|
||||
|
||||
def recover_bao():
|
||||
# Explicit operator recovery after the server leaf itself has expired.
|
||||
key, root = ca()
|
||||
leaf_key, leaf = sign_leaf(
|
||||
key,
|
||||
root,
|
||||
"openbao",
|
||||
["openbao", "localhost"],
|
||||
["127.0.0.1"],
|
||||
seconds=120,
|
||||
)
|
||||
install(
|
||||
"openbao",
|
||||
{
|
||||
"certificate": public(leaf).decode(),
|
||||
"private_key": key_bytes(leaf_key).decode(),
|
||||
"ca_chain": [public(root).decode()],
|
||||
},
|
||||
)
|
||||
time.sleep(6)
|
||||
operator = Operator()
|
||||
try:
|
||||
operator.unseal()
|
||||
for role in ROLES:
|
||||
if role != "migrate":
|
||||
repair(operator, role)
|
||||
finally:
|
||||
operator.close()
|
||||
|
||||
|
||||
def rollover():
|
||||
old_key, old_root = ca()
|
||||
new_key, new_root = make_root()
|
||||
directory = OPERATOR / ("rollover-" + uuid4().hex[:12])
|
||||
directory.mkdir(mode=0o700)
|
||||
fixture_key, fixture = sign_leaf(
|
||||
old_key, old_root, "lb", ["lb"], seconds=900
|
||||
)
|
||||
write_private(
|
||||
SECRET / "probe/retired-ca-lb/cert.pem",
|
||||
public(fixture) + public(old_root),
|
||||
mode=0o644,
|
||||
)
|
||||
write_private(
|
||||
SECRET / "probe/retired-ca-lb/key.pem",
|
||||
key_bytes(fixture_key),
|
||||
mode=0o644,
|
||||
)
|
||||
fixture_check("retired-ca-lb")
|
||||
for name, value in [
|
||||
("old-key.pem", key_bytes(old_key)),
|
||||
("old-ca.pem", public(old_root)),
|
||||
("new-key.pem", key_bytes(new_key)),
|
||||
("new-ca.pem", public(new_root)),
|
||||
]:
|
||||
write_private(directory / name, value)
|
||||
journal = {
|
||||
"phase": "dual_trust",
|
||||
"old_ca_sha256": old_root.fingerprint(hashes.SHA256()).hex(),
|
||||
"new_ca_sha256": new_root.fingerprint(hashes.SHA256()).hex(),
|
||||
}
|
||||
write_private(directory / "state.json", json.dumps(journal) + "\n")
|
||||
write_private(
|
||||
TRUST / "ca.pem", public(old_root) + public(new_root), mode=0o644
|
||||
)
|
||||
reload_lb() # Load both roots before any API server leaf changes issuer.
|
||||
reload_bao() # Native client CA requires an explicit listener restart.
|
||||
operator = Operator()
|
||||
try:
|
||||
for role in [*ROLES, "operator"]:
|
||||
operator.authorized(
|
||||
"POST",
|
||||
"auth/cert/certs/" + role + "-v2",
|
||||
{
|
||||
"display_name": role,
|
||||
"certificate": public(new_root).decode(),
|
||||
"allowed_common_names": [role],
|
||||
"allowed_uri_sans": ["spiffe://messenger/" + role],
|
||||
"token_policies": ["messenger-" + role],
|
||||
"token_ttl": "900s" if role == "operator" else "120s",
|
||||
"token_max_ttl": "3600s" if role == "operator" else "600s",
|
||||
"token_no_default_policy": True,
|
||||
},
|
||||
)
|
||||
csr = operator.authorized(
|
||||
"POST",
|
||||
"pki/intermediate/generate/internal",
|
||||
{
|
||||
"common_name": "Messenger rollover intermediate "
|
||||
+ uuid4().hex[:12],
|
||||
"key_type": "ec",
|
||||
"key_bits": 256,
|
||||
},
|
||||
)["data"]["csr"]
|
||||
intermediate = sign_intermediate(
|
||||
new_key, new_root, x509.load_pem_x509_csr(csr.encode())
|
||||
)
|
||||
imported = operator.authorized(
|
||||
"POST",
|
||||
"pki/intermediate/set-signed",
|
||||
{
|
||||
"certificate": (
|
||||
public(intermediate) + public(new_root)
|
||||
).decode()
|
||||
},
|
||||
)["data"]["imported_issuers"]
|
||||
chosen = None
|
||||
for issuer in imported:
|
||||
details = operator.authorized("GET", "pki/issuer/" + issuer)[
|
||||
"data"
|
||||
]
|
||||
if (
|
||||
x509.load_pem_x509_certificate(
|
||||
details["certificate"].encode()
|
||||
).serial_number
|
||||
== intermediate.serial_number
|
||||
):
|
||||
chosen = issuer
|
||||
assert chosen
|
||||
operator.authorized(
|
||||
"POST",
|
||||
"pki/config/issuers",
|
||||
{"default": chosen, "default_follows_latest_issuer": False},
|
||||
)
|
||||
journal["phase"] = "new_issuer_active"
|
||||
write_private(directory / "state.json", json.dumps(journal) + "\n")
|
||||
deadline = time.monotonic() + 120
|
||||
active = set(ROLES) - {"migrate"}
|
||||
while time.monotonic() < deadline:
|
||||
observed = metadata()
|
||||
if active <= observed.keys() and all(
|
||||
observed[r]["issuer"] == intermediate.subject.rfc4514_string()
|
||||
for r in active
|
||||
):
|
||||
break
|
||||
time.sleep(3)
|
||||
else:
|
||||
raise TimeoutError(
|
||||
"Not all active services renewed into new issuer"
|
||||
)
|
||||
fixture_check("retired-ca-lb")
|
||||
# Allow old L4 connections to expire, then remove old trust everywhere.
|
||||
time.sleep(46)
|
||||
for role in [*ROLES, "operator"]:
|
||||
operator.authorized("DELETE", "auth/cert/certs/" + role + "-v1")
|
||||
client_key, client_cert = sign_leaf(
|
||||
new_key, new_root, "operator", ["operator"], seconds=86400
|
||||
)
|
||||
write_private(OPERATOR / "client/key.pem", key_bytes(client_key))
|
||||
write_private(
|
||||
OPERATOR / "client/cert.pem",
|
||||
public(client_cert) + public(new_root),
|
||||
)
|
||||
write_private(OPERATOR / "ca-key.pem", key_bytes(new_key))
|
||||
write_private(OPERATOR / "ca.pem", public(new_root))
|
||||
write_private(TRUST / "ca.pem", public(new_root), mode=0o644)
|
||||
journal["phase"] = "old_trust_retired"
|
||||
write_private(directory / "state.json", json.dumps(journal) + "\n")
|
||||
finally:
|
||||
operator.close()
|
||||
reload_lb()
|
||||
reload_bao()
|
||||
fixture_check("retired-ca-lb", reject=True)
|
||||
fixture_check("lb")
|
||||
journal["phase"] = "complete"
|
||||
write_private(directory / "state.json", json.dumps(journal) + "\n")
|
||||
return {
|
||||
**journal,
|
||||
"issuer_id": chosen,
|
||||
"active_service_count": len(active),
|
||||
"old_l4_connections_waited_seconds": 46,
|
||||
"old_ca_fixture_valid_during_dual_trust": True,
|
||||
"old_ca_new_connection_denied_after_retirement": True,
|
||||
"services": metadata(),
|
||||
"image_rebuilt": False,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"action", choices=["metadata", "rollover", "recover-bao"]
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.action == "metadata":
|
||||
result = metadata()
|
||||
elif args.action == "rollover":
|
||||
result = rollover()
|
||||
else:
|
||||
recover_bao()
|
||||
result = {"recovered": True}
|
||||
print(json.dumps(result))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Operator-only, idempotent SQL roles and scoped S3 accounts."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.credentials import StaticProvider
|
||||
from minio.minioadmin import MinioAdmin
|
||||
from psycopg import sql
|
||||
|
||||
from scripts.bootstrap_security import OPERATOR, TRUST
|
||||
from scripts.stand import compose
|
||||
|
||||
|
||||
def attach_policy(client, policy, user):
|
||||
attached = json.loads(client.user_info(user)).get("policyName", "") or ""
|
||||
if policy not in attached.split(","):
|
||||
client.attach_policy([policy], user=user)
|
||||
|
||||
|
||||
def postgres(raw):
|
||||
# The bootstrap SQL channel is the server's local Unix socket. No password
|
||||
# or SQL literal is placed in argv; stdin and output are never recorded.
|
||||
statements = [
|
||||
"REVOKE ALL ON SCHEMA public FROM PUBLIC; REVOKE ALL ON DATABASE messenger FROM PUBLIC;"
|
||||
]
|
||||
grants = {
|
||||
"api": {
|
||||
"SELECT": [
|
||||
"users",
|
||||
"sessions",
|
||||
"chats",
|
||||
"members",
|
||||
"messages",
|
||||
"idempotency",
|
||||
"attachments",
|
||||
"message_attachments",
|
||||
"upload_reservations",
|
||||
"jobs",
|
||||
"outbox",
|
||||
"attachment_variants",
|
||||
"effect_publications",
|
||||
],
|
||||
"INSERT": [
|
||||
"users",
|
||||
"sessions",
|
||||
"chats",
|
||||
"members",
|
||||
"messages",
|
||||
"idempotency",
|
||||
"attachments",
|
||||
"message_attachments",
|
||||
"upload_reservations",
|
||||
"jobs",
|
||||
"outbox",
|
||||
],
|
||||
"UPDATE": ["users", "idempotency", "jobs"],
|
||||
"DELETE": ["sessions", "upload_reservations"],
|
||||
},
|
||||
"worker": {
|
||||
"SELECT": [
|
||||
"jobs",
|
||||
"attachments",
|
||||
"attachment_variants",
|
||||
"effect_publications",
|
||||
"pipeline_faults",
|
||||
],
|
||||
"INSERT": [
|
||||
"attachment_variants",
|
||||
"effect_publications",
|
||||
"pipeline_events",
|
||||
"outbox",
|
||||
],
|
||||
"UPDATE": ["jobs", "attachments", "pipeline_faults"],
|
||||
},
|
||||
"relay": {
|
||||
"SELECT": ["jobs", "outbox", "attachments"],
|
||||
"INSERT": ["pipeline_events", "outbox"],
|
||||
"UPDATE": ["jobs", "attachments", "outbox"],
|
||||
},
|
||||
"observer": {
|
||||
"SELECT": [
|
||||
"jobs",
|
||||
"effect_publications",
|
||||
"pipeline_events",
|
||||
"job_slo_observations",
|
||||
],
|
||||
"INSERT": ["job_slo_observations"],
|
||||
"UPDATE": ["job_slo_observations"],
|
||||
},
|
||||
"reconciler": {
|
||||
"SELECT": [
|
||||
"jobs",
|
||||
"attachments",
|
||||
"attachment_variants",
|
||||
"upload_reservations",
|
||||
"message_attachments",
|
||||
"effect_publications",
|
||||
],
|
||||
"UPDATE": ["jobs", "attachments"],
|
||||
"DELETE": [
|
||||
"attachments",
|
||||
"attachment_variants",
|
||||
"upload_reservations",
|
||||
"jobs",
|
||||
"outbox",
|
||||
"job_slo_observations",
|
||||
],
|
||||
},
|
||||
}
|
||||
# SQL literals use psycopg's own adapter; names are fixed local roles.
|
||||
for role, permissions in grants.items():
|
||||
name = "svc_" + role
|
||||
password = raw["services"][role]["database_password"]
|
||||
statements.append(
|
||||
"DO $$ BEGIN IF NOT EXISTS(SELECT 1 FROM pg_roles WHERE rolname='"
|
||||
+ name
|
||||
+ "') THEN CREATE ROLE "
|
||||
+ name
|
||||
+ " LOGIN; END IF; END $$;"
|
||||
)
|
||||
statements.append(
|
||||
sql.SQL(
|
||||
"ALTER ROLE {} WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD {};"
|
||||
)
|
||||
.format(sql.Identifier(name), sql.Literal(password))
|
||||
.as_string()
|
||||
)
|
||||
statements.append(
|
||||
"GRANT CONNECT ON DATABASE messenger TO "
|
||||
+ name
|
||||
+ "; GRANT USAGE ON SCHEMA public TO "
|
||||
+ name
|
||||
+ ";"
|
||||
)
|
||||
statements.append(
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM " + name + ";"
|
||||
)
|
||||
statements.append(
|
||||
"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM " + name + ";"
|
||||
)
|
||||
for action, tables in permissions.items():
|
||||
statements.append(
|
||||
"GRANT "
|
||||
+ action
|
||||
+ " ON "
|
||||
+ ",".join(tables)
|
||||
+ " TO "
|
||||
+ name
|
||||
+ ";"
|
||||
)
|
||||
# Only append-only technical event/outbox IDs use sequences.
|
||||
sequences = {
|
||||
"api": ["outbox_id_seq"],
|
||||
"worker": ["outbox_id_seq", "pipeline_events_id_seq"],
|
||||
"relay": ["outbox_id_seq", "pipeline_events_id_seq"],
|
||||
}.get(role, [])
|
||||
if sequences:
|
||||
statements.append(
|
||||
"GRANT USAGE ON SEQUENCE "
|
||||
+ ",".join(sequences)
|
||||
+ " TO "
|
||||
+ name
|
||||
+ ";"
|
||||
)
|
||||
statements.append("GRANT EXECUTE ON FUNCTION login_wait(text) TO svc_api;")
|
||||
result = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-q",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
input="\n".join(statements),
|
||||
)
|
||||
assert "ERROR" not in result
|
||||
|
||||
|
||||
def s3(raw):
|
||||
context = ssl.create_default_context(cafile=TRUST / "ca.pem")
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_3
|
||||
context.load_cert_chain(
|
||||
OPERATOR / "client/cert.pem", OPERATOR / "client/key.pem"
|
||||
)
|
||||
pool = urllib3.PoolManager(
|
||||
ssl_context=context,
|
||||
timeout=urllib3.Timeout(connect=2, read=10),
|
||||
retries=False,
|
||||
)
|
||||
credentials = StaticProvider(raw["s3_root_user"], raw["s3_root_password"])
|
||||
admin = MinioAdmin(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=credentials,
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
)
|
||||
client = Minio(
|
||||
"localhost:" + os.getenv("S3_ADMIN_PORT", "9007"),
|
||||
credentials=credentials,
|
||||
secure=True,
|
||||
http_client=pool,
|
||||
region="us-east-1",
|
||||
)
|
||||
for attempt in range(60):
|
||||
try:
|
||||
if not client.bucket_exists("messenger"):
|
||||
client.make_bucket("messenger")
|
||||
break
|
||||
except Exception:
|
||||
if attempt == 59:
|
||||
raise RuntimeError("S3 TLS bootstrap unavailable") from None
|
||||
time.sleep(1)
|
||||
for role in ["api", "worker", "reconciler"]:
|
||||
statements = [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger"],
|
||||
}
|
||||
]
|
||||
|
||||
def allow(actions, resource, condition=None, statements=statements):
|
||||
statement = {
|
||||
"Effect": "Allow",
|
||||
"Action": actions,
|
||||
"Resource": [resource],
|
||||
}
|
||||
if condition:
|
||||
statement["Condition"] = condition
|
||||
statements.append(statement)
|
||||
|
||||
allow(["s3:GetObject"], "arn:aws:s3:::messenger/objects/*")
|
||||
if role == "api":
|
||||
allow(["s3:PutObject"], "arn:aws:s3:::messenger/objects/staging/*")
|
||||
if role == "worker":
|
||||
allow(["s3:PutObject"], "arn:aws:s3:::messenger/objects/*/v*/*")
|
||||
if role == "reconciler":
|
||||
allow(["s3:DeleteObject"], "arn:aws:s3:::messenger/objects/*")
|
||||
allow(
|
||||
["s3:ListBucket"],
|
||||
"arn:aws:s3:::messenger",
|
||||
{"StringLike": {"s3:prefix": ["objects/*"]}},
|
||||
)
|
||||
values = raw["services"][role]["values"]
|
||||
admin.policy_add(
|
||||
"messenger-" + role,
|
||||
policy={"Version": "2012-10-17", "Statement": statements},
|
||||
)
|
||||
admin.user_add(values["S3_ACCESS_KEY"], values["S3_SECRET_KEY"])
|
||||
attach_policy(admin, "messenger-" + role, values["S3_ACCESS_KEY"])
|
||||
pool.clear()
|
||||
|
||||
|
||||
def main():
|
||||
raw = json.loads((OPERATOR / "service-secrets.json").read_text())
|
||||
postgres(raw)
|
||||
s3(raw)
|
||||
print("SQL role grants and scoped S3 accounts prepared; no values printed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,636 @@
|
||||
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
from scripts.tls_client import FreshTrustTransport
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, queries, spans, wait
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
|
||||
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
|
||||
SIGNALS = {
|
||||
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
|
||||
"queue_depth": "max(messenger_queue_depth)",
|
||||
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
|
||||
"worker_busy": "sum(messenger_worker_busy)",
|
||||
"worker_attempts": "sum(messenger_job_attempts_total)",
|
||||
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
|
||||
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
|
||||
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
|
||||
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
|
||||
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
|
||||
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
|
||||
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
|
||||
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
|
||||
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
|
||||
"collector_queue": "sum(otelcol_exporter_queue_size)",
|
||||
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
|
||||
"collector_memory": 'otelcol_process_memory_rss{job="collector"}',
|
||||
"app_memory": "sum(messenger_process_memory_bytes)",
|
||||
}
|
||||
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
|
||||
|
||||
|
||||
def snapshot(phase):
|
||||
observed = queries(SIGNALS)
|
||||
values = {}
|
||||
for key, rows in observed.items():
|
||||
value = float(rows[0]["value"][1]) if rows else None
|
||||
values[key] = (
|
||||
value if value is not None and math.isfinite(value) else None
|
||||
)
|
||||
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
|
||||
return {
|
||||
"at": time.time(),
|
||||
"phase": phase,
|
||||
"signals": values,
|
||||
"alerts": [
|
||||
{
|
||||
"name": a["labels"]["alertname"],
|
||||
"state": a["state"],
|
||||
"active_at": a.get("activeAt"),
|
||||
}
|
||||
for a in alerts
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def firing(samples):
|
||||
return {
|
||||
a["name"]
|
||||
for sample in samples
|
||||
for a in sample["alerts"]
|
||||
if a["state"] == "firing"
|
||||
}
|
||||
|
||||
|
||||
class Workload:
|
||||
def __init__(self, client, headers, chat):
|
||||
self.client, self.headers, self.chat = client, headers, chat
|
||||
self.stop = threading.Event()
|
||||
self.phase = "warmup"
|
||||
self.requests = []
|
||||
self.uploads = []
|
||||
self.lock = threading.Lock()
|
||||
self.slots = threading.BoundedSemaphore(16)
|
||||
self.pool = ThreadPoolExecutor(max_workers=16)
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
|
||||
self.raw = image.getvalue()
|
||||
self.threads = [
|
||||
threading.Thread(target=self.schedule_messages),
|
||||
threading.Thread(target=self.schedule_uploads),
|
||||
]
|
||||
|
||||
def write(self):
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/messages",
|
||||
headers={
|
||||
**self.headers,
|
||||
"Idempotency-Key": uuid4().hex,
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
json={"text": "Synthetic SLO message " + uuid4().hex},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 201:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
finally:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[
|
||||
started,
|
||||
phase,
|
||||
status,
|
||||
time.time() - started,
|
||||
trace_id,
|
||||
identity,
|
||||
]
|
||||
)
|
||||
self.slots.release()
|
||||
|
||||
def schedule_messages(self):
|
||||
next_at = time.monotonic()
|
||||
while not self.stop.is_set():
|
||||
if self.slots.acquire(blocking=False):
|
||||
self.pool.submit(self.write)
|
||||
else:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[time.time(), self.phase, -1, 0, "", ""]
|
||||
)
|
||||
next_at += 0.25
|
||||
self.stop.wait(max(0, next_at - time.monotonic()))
|
||||
|
||||
def schedule_uploads(self):
|
||||
while not self.stop.is_set():
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/attachments",
|
||||
params={"filename": "synthetic-slo.png"},
|
||||
content=self.raw,
|
||||
headers={
|
||||
**self.headers,
|
||||
"Content-Type": "image/png",
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 202:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
with self.lock:
|
||||
self.uploads.append(
|
||||
[started, phase, status, identity, trace_id]
|
||||
)
|
||||
self.stop.wait(max(0, 5 - (time.time() - started)))
|
||||
|
||||
def start(self):
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
for thread in self.threads:
|
||||
thread.join(20)
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/slo-research-first.json")
|
||||
parser.add_argument("--minimum-seconds", type=int, default=1800)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists(), "Choose a new output path"
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"scope": "full30m"
|
||||
if args.minimum_seconds >= 1800
|
||||
else "short-development",
|
||||
"profile": {
|
||||
"messages_per_second": 4,
|
||||
"max_inflight": 16,
|
||||
"upload_every_seconds": 5,
|
||||
"image_pixels": [320, 200],
|
||||
},
|
||||
"complete": False,
|
||||
"samples": [],
|
||||
"phases": [],
|
||||
}
|
||||
study_started = time.monotonic()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load = None
|
||||
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
|
||||
with httpx.Client(
|
||||
transport=FreshTrustTransport(),
|
||||
base_url=BASE,
|
||||
timeout=8,
|
||||
trust_env=False,
|
||||
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
|
||||
) as client:
|
||||
ready(timeout=90)
|
||||
config = json.loads(compose("config", "--format", "json"))
|
||||
report["container_limits"] = {
|
||||
name: {
|
||||
k: int(service[k])
|
||||
if k in {"mem_limit", "memswap_limit", "scale"}
|
||||
else service[k]
|
||||
for k in ["cpus", "mem_limit", "memswap_limit", "scale"]
|
||||
if k in service
|
||||
}
|
||||
for name, service in config["services"].items()
|
||||
}
|
||||
report["docker_vm"] = json.loads(
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}',
|
||||
]
|
||||
)
|
||||
)
|
||||
report["runtime_versions"] = sorted(
|
||||
{
|
||||
row["metric"].get("service_version", "unknown")
|
||||
for row in queries(
|
||||
{"versions": "messenger_process_memory_bytes"}
|
||||
)["versions"]
|
||||
}
|
||||
)
|
||||
user, _, session = actor(client)
|
||||
headers = auth(session)
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Synthetic SLO experiment",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
chat = response.json()["id"]
|
||||
load = Workload(client, headers, chat)
|
||||
workload = load
|
||||
|
||||
def save():
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
|
||||
def hold(name, seconds, *, until_clear=None, max_seconds=None):
|
||||
load.phase = name
|
||||
phase = {
|
||||
"name": name,
|
||||
"start": time.time(),
|
||||
"planned_seconds": seconds,
|
||||
"maximum_seconds": max_seconds or seconds,
|
||||
}
|
||||
report["phases"].append(phase)
|
||||
minimum_end = time.monotonic() + seconds
|
||||
deadline = time.monotonic() + (max_seconds or seconds)
|
||||
start_index = len(report["samples"])
|
||||
while time.monotonic() < deadline:
|
||||
sample = snapshot(name)
|
||||
report["samples"].append(sample)
|
||||
save()
|
||||
if (
|
||||
until_clear is not None
|
||||
and time.monotonic() >= minimum_end
|
||||
and not until_clear.intersection(firing([sample]))
|
||||
):
|
||||
break
|
||||
time.sleep(min(5, max(0, deadline - time.monotonic())))
|
||||
phase["end"] = time.time()
|
||||
phase["firing"] = sorted(firing(report["samples"][start_index:]))
|
||||
return phase
|
||||
|
||||
def db_delay(on):
|
||||
if on:
|
||||
sql(
|
||||
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
|
||||
)
|
||||
else:
|
||||
sql(
|
||||
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
|
||||
)
|
||||
|
||||
try:
|
||||
load.start()
|
||||
warm = hold("warmup", 300)
|
||||
settled = [
|
||||
s for s in report["samples"] if s["at"] >= warm["end"] - 60
|
||||
]
|
||||
assert not BURN_ALERTS.intersection(firing(settled)), (
|
||||
"Baseline is not clean"
|
||||
)
|
||||
assert (
|
||||
report["samples"][-1]["signals"]["collector_memory"]
|
||||
is not None
|
||||
)
|
||||
report["baseline"] = snapshot("baseline")
|
||||
for signal in ["api", "ready"]:
|
||||
if signal == "api":
|
||||
activate, restore = (
|
||||
lambda: db_delay(True),
|
||||
lambda: db_delay(False),
|
||||
)
|
||||
else:
|
||||
|
||||
def activate():
|
||||
return compose("stop", "--timeout", "1", "worker")
|
||||
|
||||
def restore():
|
||||
return compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "worker"
|
||||
)
|
||||
|
||||
brief_started = time.time()
|
||||
activate()
|
||||
brief = hold(signal + "_brief", 5)
|
||||
restore()
|
||||
settle = hold(
|
||||
signal + "_brief_recovery", 120 if signal == "api" else 60
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(
|
||||
brief["firing"] + settle["firing"]
|
||||
), "Brief incident triggered paging"
|
||||
report.setdefault("short_spikes", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": brief_started,
|
||||
"paging_events": 0,
|
||||
}
|
||||
)
|
||||
action_started = time.time()
|
||||
activate()
|
||||
action_finished = time.time()
|
||||
degraded = hold(
|
||||
signal + "_sustained", 180 if signal == "api" else 150
|
||||
)
|
||||
restore_started = time.time()
|
||||
restore()
|
||||
restore_finished = time.time()
|
||||
expected = (
|
||||
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
|
||||
if signal == "api"
|
||||
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
|
||||
)
|
||||
recovered = hold(
|
||||
signal + "_recovery",
|
||||
150 if signal == "api" else 240,
|
||||
until_clear=expected,
|
||||
max_seconds=600,
|
||||
)
|
||||
assert expected <= set(degraded["firing"]), (
|
||||
signal,
|
||||
degraded["firing"],
|
||||
)
|
||||
last = report["samples"][-1]
|
||||
assert not expected.intersection(firing([last])), (
|
||||
"Incident alerts did not resolve"
|
||||
)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if action_finished <= s["at"] <= recovered["end"]
|
||||
]
|
||||
timings = {}
|
||||
for name in sorted(expected):
|
||||
first = next(
|
||||
s["at"] for s in points if name in firing([s])
|
||||
)
|
||||
resolved = next(
|
||||
s["at"]
|
||||
for s in points
|
||||
if s["at"] >= restore_finished
|
||||
and name not in firing([s])
|
||||
)
|
||||
timings[name] = {
|
||||
"detect_seconds": round(first - action_finished, 3),
|
||||
"resolve_seconds": round(
|
||||
resolved - restore_finished, 3
|
||||
),
|
||||
}
|
||||
report.setdefault("incidents", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": action_started,
|
||||
"action_finished": action_finished,
|
||||
"restore_started": restore_started,
|
||||
"restore_finished": restore_finished,
|
||||
"timings": timings,
|
||||
}
|
||||
)
|
||||
if signal == "api":
|
||||
candidates = [
|
||||
r
|
||||
for r in load.requests
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] >= 500
|
||||
]
|
||||
assert candidates
|
||||
items = wait(
|
||||
lambda identity=candidates[0][4]: spans(identity)
|
||||
)
|
||||
assert any(
|
||||
s["name"] == "db.postgresql"
|
||||
and s["duration_seconds"] >= 2
|
||||
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
||||
for s in items
|
||||
)
|
||||
report["database_diagnostic_trace"] = items
|
||||
else:
|
||||
assert (
|
||||
max(
|
||||
s["signals"]["oldest_job_age"] or 0 for s in points
|
||||
)
|
||||
> 20
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["queue_depth"] or 0 for s in points)
|
||||
> 0
|
||||
)
|
||||
accepted = [
|
||||
r
|
||||
for r in load.uploads
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] == 202
|
||||
]
|
||||
assert accepted
|
||||
report["worker_recovered_trace"] = wait(
|
||||
lambda identity=accepted[0][4]: spans(identity)
|
||||
)
|
||||
report["worker_recovered_attachment"] = accepted[0][3]
|
||||
outage_before = snapshot("backend_before")
|
||||
outage_started = time.time()
|
||||
compose("stop", "--timeout", "1", "loki", "tempo")
|
||||
outage = hold("backend_unavailable", 90)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if s["phase"] == "backend_unavailable"
|
||||
]
|
||||
assert max(
|
||||
s["signals"]["collector_failed_spans"] or 0 for s in points
|
||||
) > (outage_before["signals"]["collector_failed_spans"] or 0)
|
||||
assert all(
|
||||
s["signals"]["collector_queue"] is not None
|
||||
and s["signals"]["collector_queue_capacity"] is not None
|
||||
and s["signals"]["collector_memory"] is not None
|
||||
for s in points
|
||||
)
|
||||
assert all(
|
||||
(s["signals"]["collector_queue"] or 0)
|
||||
<= (s["signals"]["collector_queue_capacity"] or 0)
|
||||
for s in points
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["collector_memory"] for s in points)
|
||||
< report["container_limits"]["otel"]["mem_limit"]
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(outage["firing"])
|
||||
during = [
|
||||
r for r in load.requests if r[1] == "backend_unavailable"
|
||||
]
|
||||
assert during and all(r[2] == 201 for r in during)
|
||||
before_ready = sql(
|
||||
"SELECT count(*) FROM attachments WHERE state='ready'"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
|
||||
wait(lambda: fetch("loki", "/ready", json_result=False))
|
||||
wait(lambda: fetch("tempo", "/ready", json_result=False))
|
||||
hold("backend_recovery", 60)
|
||||
restored = next(
|
||||
r
|
||||
for r in reversed(load.uploads)
|
||||
if r[1] == "backend_recovery" and r[2] == 202
|
||||
)
|
||||
wait(
|
||||
lambda: any(
|
||||
s["name"] == "attachment.process"
|
||||
for s in spans(restored[4])
|
||||
)
|
||||
)
|
||||
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
|
||||
report["backend_outage"] = {
|
||||
"started": outage_started,
|
||||
"finished": outage["end"],
|
||||
"before": outage_before,
|
||||
"after": snapshot("backend_after"),
|
||||
"ready_count_before_backend_recovery": int(before_ready),
|
||||
"successful_writes_during_outage": len(during),
|
||||
"restored_trace_id": restored[4],
|
||||
"queue_bounded": True,
|
||||
"collector_rss_below_limit": True,
|
||||
"send_failure_counter_counts_attempts_not_unique_loss": True,
|
||||
}
|
||||
remaining = args.minimum_seconds - (
|
||||
time.monotonic() - study_started
|
||||
)
|
||||
if remaining > 0:
|
||||
hold("steady_after_recovery", remaining)
|
||||
load.close()
|
||||
load = None
|
||||
# Read through the public API: a successful SQL insert alone is not
|
||||
# proof that every acknowledged message remains accessible.
|
||||
seen, cursor = set(), None
|
||||
while True:
|
||||
page = client.get(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers=headers,
|
||||
params={
|
||||
"limit": 100,
|
||||
**({"cursor": cursor} if cursor else {}),
|
||||
},
|
||||
)
|
||||
page.raise_for_status()
|
||||
data = page.json()
|
||||
ids = [row["id"] for row in data["items"]]
|
||||
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
|
||||
seen.update(ids)
|
||||
cursor = data.get("next_cursor")
|
||||
if not cursor:
|
||||
break
|
||||
expected_messages = {
|
||||
r[5] for r in workload.requests if r[2] == 201
|
||||
}
|
||||
assert seen == expected_messages
|
||||
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
|
||||
for row in accepted_uploads:
|
||||
|
||||
def completed(identity=row[3]):
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{identity}", headers=headers
|
||||
)
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["status"] == "ready"
|
||||
)
|
||||
|
||||
wait(completed, timeout=120)
|
||||
for row in accepted_uploads[
|
||||
:: max(1, len(accepted_uploads) // 10)
|
||||
]:
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{row[3]}/content", headers=headers
|
||||
)
|
||||
assert (
|
||||
response.status_code == 200
|
||||
and response.content == workload.raw
|
||||
)
|
||||
report["preservation"] = {
|
||||
"acknowledged_messages": len(expected_messages),
|
||||
"accessible_messages": len(seen),
|
||||
"accepted_uploads": len(accepted_uploads),
|
||||
"all_accepted_uploads_ready": True,
|
||||
"sampled_originals_match": True,
|
||||
}
|
||||
report["rules"] = fetch("prometheus", "/api/v1/rules")
|
||||
report["final_30m_sli"] = queries(
|
||||
{
|
||||
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
|
||||
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
|
||||
}
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
try:
|
||||
db_delay(False)
|
||||
finally:
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"worker",
|
||||
"loki",
|
||||
"tempo",
|
||||
)
|
||||
if load is not None:
|
||||
load.close()
|
||||
report["elapsed_seconds"] = round(
|
||||
time.monotonic() - study_started, 3
|
||||
)
|
||||
request_path = output.with_name(output.stem + "-requests.csv")
|
||||
with request_path.open("w") as stream:
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(
|
||||
[
|
||||
"timestamp",
|
||||
"phase",
|
||||
"status",
|
||||
"seconds",
|
||||
"trace_id",
|
||||
"message_id",
|
||||
]
|
||||
)
|
||||
writer.writerows(workload.requests)
|
||||
report["requests_csv"] = request_path.name
|
||||
report["request_statuses"] = dict(
|
||||
Counter(str(r[2]) for r in workload.requests)
|
||||
)
|
||||
report["uploads"] = workload.uploads
|
||||
save()
|
||||
print(
|
||||
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 -m scripts.restart_secure
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user