ЛР 7 — БВТ2301 — Курлова — на 4 #17

Closed
alenccha wants to merge 1 commits from alenccha/mtusi2026lab7:solution/bvt2301-kurlova into main
116 changed files with 14522 additions and 63 deletions
+48 -22
View File
@@ -1,41 +1,67 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=https://s3:9000
GRACEFUL_TIMEOUT=8
MAX_INFLIGHT_REQUESTS=64
MAX_BODY_BYTES=65536
MAX_RSS_MB=0
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_TIMEOUT=3
DB_CONNECT_TIMEOUT=3
DB_STATEMENT_TIMEOUT_MS=5000
DB_OP_TIMEOUT=6
REQUEST_TIMEOUT=30
DB_READY_TIMEOUT=1.5
READY_CHECK_SECONDS=1
READY_FAILURES=2
SESSION_CACHE_TTL_SECONDS=30
SESSION_SWEEP_SECONDS=60
CACHE_TIMEOUT=0.2
SESSION_REVOKED_TTL_SECONDS=60
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
UPLOAD_CONCURRENCY=2
RECONCILE_INTERVAL_SECONDS=300
RECONCILE_GRACE_SECONDS=900
# --- lab5: broker, jobs, worker
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=20
JOB_BACKOFF_BASE_SECONDS=2
JOB_BACKOFF_MAX_SECONDS=30
JOB_SWEEP_SECONDS=5
WORKER_CHILD_MEMORY_MB=1024
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing.
MAX_PENDING_JOBS=200
JPEG_QUALITY=85
WEBP_QUALITY=80
# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else.
FAULT_INJECTION=1
# Session store: PostgreSQL is the source of truth, Valkey (TLS + ACL user `api`) caches sessions and holds the
# shared login rate limit counters.
SESSION_STORE=valkey
# --- lab7: TLS and OpenBao. No passwords or tokens here: compose.yaml passes only file PATHS (Docker secrets).
PUBLIC_BASE_URL=https://localhost:8443
TLS_CA_FILE=/run/secrets/ca.crt
TLS_CERT_FILE=/run/secrets/service.crt
TLS_KEY_FILE=/run/secrets/service.key
BAO_ADDR=https://openbao:8200
BAO_CACERT=/run/secrets/bao-ca.crt
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
# Argon2id (RFC 9106 / OWASP minimum: m=19 MiB, t=2, p=1). Measured on the stand, see REPORT.md.
ARGON2_MEMORY_KIB=19456
ARGON2_ITERATIONS=2
ARGON2_PARALLELISM=1
# Concurrent password hashes per replica (login DoS budget) and login attempts per window, shared via Valkey.
PASSWORD_HASH_CONCURRENCY=4
LOGIN_RATE_LIMIT_PER_USER=5
LOGIN_RATE_LIMIT_PER_IP=120
LOGIN_RATE_WINDOW_SECONDS=60
# OpenBao: secrets are re-read on this period (rotation pick-up); startup waits this long for an unsealed OpenBao.
BAO_REFRESH_SECONDS=15
BAO_STARTUP_TIMEOUT_SECONDS=120
+48
View File
@@ -0,0 +1,48 @@
name: Application stand (lab7)
# Student job: the whole operator bootstrap in a throwaway runner, then the public HTTPS checks and the TLS audit.
# The job itself plays the operator: PKI, bootstrap passwords and OpenBao shares are created inside the runner and
# destroyed with it. No repository secrets are used; pull_request_target is deliberately not a trigger.
on: [push, pull_request]
permissions:
contents: read
jobs:
stand:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Local PKI and bootstrap credentials (operator step)
run: |
sh scripts/pki.sh init
sh scripts/init-secrets.sh
- name: Validate Compose configuration
run: docker compose --env-file .env config --quiet
- name: OpenBao init, unseal and bootstrap (operator step)
run: |
docker compose --env-file .env up -d openbao
sh scripts/bao.sh init
sh scripts/bao.sh bootstrap
- name: Build and start the stand, wait for readiness
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
- name: Contract structure and public HTTPS checks
run: |
make check
make test CA_FILE=$PWD/secrets/pki/ca.crt
make test-security CA_FILE=$PWD/secrets/pki/ca.crt
- name: TLS / mTLS / authorization audit of the trust matrix
run: >
docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro
-v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py
- name: Container hardening from docker inspect
run: python3 student-tests/check_hardening.py
- name: Service state and logs on failure (no secrets are logged)
if: failure()
run: |
docker compose ps -a
docker compose logs --tail 60 nginx api-1 api-2 worker openbao rabbitmq postgres
- name: Remove the stand
if: always()
run: docker compose down -v
+35
View File
@@ -0,0 +1,35 @@
FROM python:3.12.16-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=120 \
PIP_RETRIES=10
RUN apt-get update \
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-deps -r requirements.txt
COPY app/ app/
COPY migrations/ migrations/
COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "app.serve"]
+190 -17
View File
@@ -1,37 +1,210 @@
# Отчёт — лабораторная 7
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Алёна Курлова, БВТ2301
- Целевая оценка; выбранный трек 5: **4** (трек 5 не заявлен)
- Commit/PR: после публикации (ветка `solution/bvt2301-kurlova`)
- Источники, библиотеки, помощь ИИ: OpenBao 2.6.5 ([AppRole/cert auth](https://openbao.org/docs/auth/cert/), [policies](https://openbao.org/docs/concepts/policies/), [production hardening](https://openbao.org/docs/internals/security/)), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [OWASP File Upload](https://cheatsheetseries.owasp.org/cheatsheets/File_Upload_Cheat_Sheet.html), [Docker security](https://docs.docker.com/engine/security/), RabbitMQ [TLS](https://www.rabbitmq.com/docs/ssl) и [access control](https://www.rabbitmq.com/docs/access-control), PostgreSQL [SSL](https://www.postgresql.org/docs/17/ssl-tcp.html), argon2-cffi, Trivy 0.58.1. Код lab6 перенесён, бизнес-API не менялся. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора. Все результаты ниже получены на локальном стенде и сохранены в `evidence/`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Desktop (Compose v2.20+), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM 10 CPU / 7.7 GiB.
**Шаги оператора** (секреты и ключи создаются локально, в Git и образы не попадают; скрипты ничего секретного не печатают):
```sh
sh scripts/pki.sh init # CA (Root + Ingest + Telemetry Clients) и leaf-сертификаты -> secrets/pki/ (0700)
sh scripts/init-secrets.sh # .env и bootstrap-пароли хранилищ -> secrets/bootstrap/ (0700)
docker compose --env-file .env up -d openbao
sh scripts/bao.sh init # 3 доли unseal (порог 2) + root token -> secrets/operator/ (0600); затем unseal
sh scripts/bao.sh bootstrap # KV v2 `messenger/`, политики, cert auth (роли api/worker), секреты сервисов
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
make check
make test CA_FILE=$PWD/secrets/pki/ca.crt
make test-security CA_FILE=$PWD/secrets/pki/ca.crt
```
**Bootstrap, повторяемый при каждом `up`:**
- `migrate` — SQL-миграции, роли `app_api`/`app_worker` и их пароли из `secrets/bootstrap`; подключение по TLS `verify-full`;
- `minio-init` — бакет, политики, пользователи `messenger-api` и `messenger-worker`; по HTTPS;
- RabbitMQ при каждом старте импортирует пользователей, права и топологию из `infra/rabbitmq/definitions.json`;
- Grafana подхватывает provisioning.
**Что делает оператор после рестарта OpenBao:** `sh scripts/bao.sh unseal`. Уже работающие сервисы продолжают работать, перезапущенные ждут unseal. Restart hook (`scripts/restart.sh`) делает unseal сам, потому что его запускает оператор.
**Проверки:**
```sh
docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro \
-v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py # вся матрица доверия, 135 проверок
python3 student-tests/test_auth_acl.py # ACL, X-User-Id, logout/expiry, лимит входа, размеры, rehash
python3 student-tests/test_cert_rotation.py # ротация и отзыв leaf lb-client, просроченный сертификат
python3 student-tests/check_hardening.py # docker inspect всех контейнеров
python3 student-tests/test_observability.py # трассы/логи/метрики через mTLS, отсутствие утечек
python3 student-tests/check_dashboard.py # все запросы dashboard, включая выполнение через Grafana
python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5
sh student-tests/operator_checks.sh # ОПЕРАТОР: ротация пароля worker, seal/unseal, persistence, failover
```
CI: `.github/workflows/stand.yml` повторяет всю последовательность оператора в одноразовом runner (PKI, OpenBao init/bootstrap, стенд, `make test`, `make test-security`, TLS-аудит, hardening). Локально эти шаги проверены; на runner workflow ещё не выполнялся.
Для LOGIN DoS-замера: `LOGIN_RATE_LIMIT_PER_IP=1000000 docker compose --env-file .env up -d --no-deps api-1 api-2 && python3 student-tests/login_budget.py`, затем вернуть лимит обычным `up`.
**Остановка и удаление.** Безопасная остановка с сохранением данных: `docker compose stop`. Удаление учебных данных: `docker compose down -v`. После этого OpenBao нужно инициализировать заново, а `secrets/operator/` устареет.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
клиент ──TLS :8443──► nginx ──mTLS (lb.client.messenger; API проверяет SAN и отзыв)──► api-1, api-2
api ─AMQPS mTLS (EXTERNAL=CN, только publish)─► RabbitMQ ─► worker (только consume)
api, worker ──TLS verify-full──► PostgreSQL (роли app_api / app_worker)
api, worker ──HTTPS──► MinIO (пользователи messenger-api / messenger-worker, права по префиксам)
api ──TLS + ACL──► Valkey (кэш сессий + общий лимит входа)
api, worker ──HTTPS + cert auth──► OpenBao (KV: только свой путь, токен 15 мин)
api, worker ──mTLS (Ingest CA)──► Collector ──mTLS (Telemetry Clients CA)──► Prometheus / Loki / Tempo ◄── Grafana
оператор ──docker compose exec / 127.0.0.1 (HTTPS, mTLS)──► OpenBao, Grafana, Prometheus/Loki/Tempo
```
**Источник истины и транзакции** (как в lab5–lab6):
- PostgreSQL — источник истины;
- upload в одной транзакции фиксирует вложение, задание и outbox и только потом отвечает 202;
- worker публикует результат одной транзакцией;
- брокер переносит только ссылки на задания;
- OpenBao в бизнес-транзакции не входит: секреты читаются при старте и обновляются фоном.
**Изменения относительно lab6:**
| Область | Что сделано | Где |
| --- | --- | --- |
| PKI | Root CA и два промежуточных: Ingest (только клиенты Collector) и Telemetry Clients (только клиенты хранилищ телеметрии); leaf 90 дн., EC P-256; перевыпуск, отзыв по serial, тестовые просроченный и чужой сертификаты | `scripts/pki.sh` |
| Внешний TLS | nginx на `127.0.0.1:8443`, HSTS, TLS 1.2+ | `infra/nginx.conf` |
| LB → API | mTLS: nginx предъявляет `lb.client.messenger` и проверяет сервер по `api.internal`. API (`app/serve.py`): CERT_REQUIRED, allowlist DNS SAN, denylist отозванных serial (перечитывается на лету), проверка на каждом соединении. Plaintext-listener нет, health только `127.0.0.1:8081` | `app/serve.py`, `app/main.py` |
| Связи с хранилищами | PostgreSQL `verify-full` + `pg_hba` только `hostssl`; MinIO HTTPS; RabbitMQ AMQPS с обязательным клиентским сертификатом; Valkey TLS-only + ACL | `infra/postgres`, `infra/minio`, `infra/rabbitmq`, `infra/valkey`, `app/db.py`, `app/s3.py`, `app/broker.py`, `app/sessions.py` |
| Минимальные права | роли БД, S3-политики по префиксам, права RabbitMQ (API: write `^attachments$`; worker: read `^attachments\.process$`; configure ни у кого, топологию создаёт оператор) | `migrations/0007_service_roles.sql`, `infra/minio/policy-*.json`, `definitions.json` |
| OpenBao | server mode, хранилище `file` на volume, TLS; cert auth: роль = точный CN, токен 15 мин / max 1 ч; политики «только свой путь»; клиент с renew/re-login и подхватом новой версии KV без рестарта | `infra/openbao/`, `scripts/bao.sh`, `app/openbao.py` |
| Телеметрия | Collector принимает только Ingest CA; Prometheus, Loki и Tempo — только Telemetry Clients CA; Grafana обращается к источникам с клиентским сертификатом; scrape-заданий нет; внутренний gRPC и self-метрики только на loopback | `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` |
| Аутентификация | Argon2id с параметрами из env (замер ниже); rehash при входе; ≤ 4 одновременных Argon2 на реплику; лимит входа 5 неудач на username и 120 попыток на IP за 60 с — общий для реплик (Valkey), 429 + Retry-After | `app/passwords.py`, `app/main.py`, `app/sessions.py` |
| Hardening | все 15 контейнеров non-root, `cap_drop: ALL`, `no-new-privileges`, read-only rootfs + tmpfs, лимиты CPU/RAM/PIDs; нет Docker socket и host network; наружу только `127.0.0.1` | `compose.yaml` |
| Обновления по scan | `python:3.12.16-slim`, Pillow 13.0.0, urllib3 2.8.0, nginx 1.30.6, OpenBao 2.6.5; guard использует `scope["path"]` вместо `request.url.path` | `Dockerfile`, `requirements.txt`, `compose.yaml` |
**Роли, версии, health, volumes, порты, credentials.** Сеть одна — внутренняя сеть Compose. Digest всех образов закреплён в `compose.yaml`. Каждый ключ попадает только в свой контейнер как Docker secret.
| Роль | Сервис | Образ | Health | Volume | Порт (наружу) | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| LB | `nginx` | nginx:1.30.6-alpine | `curl https://localhost:8443/health/live` | — | `127.0.0.1:8443` | lb-server, lb-client |
| API ×2 | `api-1`, `api-2` | сборка, python:3.12.16-slim | loopback `127.0.0.1:8081/health/ready` | — | — | api-N (сервер), api (клиент), api-otel; **паролей нет**, всё из OpenBao |
| Worker ×2 | `worker` | та же сборка | heartbeat-файл | — | — | worker (клиент), worker-otel; паролей нет |
| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | — | postgres (сервер), пароль superuser (только migrate/оператор) |
| S3 | `minio` | madlabcg/minio (digest) | `mc ready` по HTTPS | `miniodata` | — | minio (сервер), root (только minio-init) |
| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_running` | `rabbitmqdata` | — | rabbitmq (сервер); пользователей с паролями нет |
| Кэш и лимиты | `valkey` | valkey:8.1-alpine | `PING` пользователем `health` | — (кэш) | — | valkey (сервер), пароль ACL `api` |
| Секреты | `openbao` | openbao:2.6.5 | `bao status` | `baodata` | — | openbao (сервер); root token и доли у оператора |
| Telemetry | `otel-collector`, `prometheus`, `loki`, `tempo` | версии lab6 | promtool / TCP | `promdata`, `lokidata`, `tempodata` | `127.0.0.1:9090/3100/3200` (mTLS operator) | свои серверные и клиентские сертификаты |
| UI | `grafana` | grafana:11.5.2 | `curl https://grafana:3000/api/health` | `grafanadata` | `127.0.0.1:3000` (HTTPS + login) | grafana, grafana-out, admin-пароль |
**Соответствие `.env.example`:**
- `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `S3_*_FILE`, `BROKER_URL_FILE` для сервисов больше не используются: эти значения приходят из OpenBao (`messenger/api`, `messenger/worker`);
- `BROKER_URL` = `amqps://rabbitmq:5671/messenger?auth=external` — без пароля;
- `TLS_CA_FILE`, `TLS_CLIENT_CERT_FILE` и `TLS_CLIENT_KEY_FILE` (вместо `TLS_CERT_FILE`/`TLS_KEY_FILE` из примера), `TLS_SERVER_*`, `BAO_ADDR` задаются в `compose.yaml`;
- `ARGON2_*` = 19456 KiB / 2 / 1;
- добавлены `PASSWORD_HASH_CONCURRENCY`, `LOGIN_RATE_LIMIT_*`, `BAO_REFRESH_SECONDS` и `BAO_STARTUP_TIMEOUT_SECONDS`.
**Документы:** модель угроз — [security/THREAT_MODEL.md](security/THREAT_MODEL.md) (14 угроз с тестами и разбор scan); матрица доверия — [contracts/trust-matrix.md](contracts/trust-matrix.md); инструкция оператора OpenBao — [infra/openbao/README.md](infra/openbao/README.md).
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| 3: прежние возможности через HTTPS с проверкой CA и SAN | `make check`, `make validate`, `make test`, `make test-security` | все PASS; пустой trust store → ошибка проверки сертификата | `evidence/smoke_check_validate_security.txt`, `evidence/smoke.txt` |
| 3: persistence через HTTPS | `make test-persistence` (hook перезапускает все хранилища, включая OpenBao с unseal, и сервисы) | PASS; uptime всех контейнеров сброшен | `evidence/persistence.txt` |
| 3: отказ реплики через HTTPS | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 1.27 и 0.67 с | `evidence/failover.txt` |
| 3: минимум lab5–lab6 | `test_worker_down.py`, `test_failure_windows.py` (A–I), `variants_check.py`, `test_observability.py` | все PASS | `evidence/lab6_worker_down.txt`, `evidence/lab5_failure_windows.txt`, `evidence/variants_limits.txt`, `evidence/observability.txt` |
| 3: Argon2id ≥ 19 MiB, t=2, p=1, параметры обоснованы замером | `login_budget.py` (bench в контейнере API, 0.5 CPU); `test_auth_acl.py` | PHC-строка `$argon2id$v=19$m=19456,t=2,p=1$…` с солью; 11 мс на хеш; таблица ниже | `evidence/login_budget.json`, `evidence/auth_acl.txt` |
| 3: LB → API mTLS, отказ без сертификата, с чужим CA, с неверной identity | аудит A | 14/14: без сертификата, чужой CA с именем lb, `intruder`/`worker`/`api` от того же CA → соединение закрыто; `lb` → 200; plaintext → нет ответа | `evidence/tls_audit_final.txt` |
| 3: нет обхода публичным портом | аудит A, K; `check_hardening.py` | порты API не опубликованы; health 8081 закрыт из сети | `evidence/tls_audit_side_channels.txt`, `evidence/hardening.txt` |
| 3: TLS с проверкой сервера на всех связях, раздельные учётные данные | аудит B, C, D, E, F, G, H, I, J | чужой CA как trust anchor, неверный hostname, plaintext → отказ на каждой связи | `evidence/tls_audit_final.txt` |
| 3: OpenBao не dev-mode, persistent, сервисная аутентификация, scoped policy, без root token в app | аудит H; `bao.sh`; persistence | cert auth: только своя роль, токен ≤ 15 мин; чужой путь, запись, политика, admin-пути → 403; данные пережили рестарт OpenBao | `evidence/tls_audit_final.txt`, `evidence/persistence.txt` |
| 3: контейнеры без privileged/host network/socket, non-root, без лишних capabilities, лимиты; наружу только нужное | `check_hardening.py` (`docker inspect`) | 15 контейнеров, 0 проблем; опубликованы только 8443, 3000, 9090, 3100, 3200 на 127.0.0.1 | `evidence/hardening.txt`, `evidence/hardening.json` |
| 3: секреты не в Git, образах, telemetry | Trivy secret scan; `test_observability.py` | в слоях образа секретов 0; пароль, токен, текст, filename, Authorization, PNG в трассах и логах не найдены | `evidence/image_scan.txt`, `evidence/observability.txt` |
| 3: лимит входа 429 + Retry-After, размер запросов, посторонний, logout/expiry | `test_auth_acl.py` | 404 постороннему на всё; поддельный X-User-Id игнорируется; logout и expiry → 401 на обеих репликах; 70 kB JSON и > 10 MiB upload → 413; лимит входа → 429 | `evidence/auth_acl.txt` |
| 3: локальный bootstrap сертификатов | `scripts/pki.sh init` | CA и leaf создаются в `secrets/pki/`, ключи не в Git | `scripts/pki.sh` |
| 4: mTLS к брокеру и Collector, разные identities, чужой сертификат того же CA не даёт прав | аудит D, E | RabbitMQ: `intruder`, `lb` (тот же CA) → ACCESS_REFUSED; API не может consume и declare; worker не может publish. Collector: `lb`, `api` (Root), `grafana-out` → отказ; api-otel, worker-otel → 200 | `evidence/tls_audit_final.txt` |
| 4: хеши и ошибки не раскрывают credentials; rehash; бюджет login DoS; общий rate limit | `test_auth_acl.py`, `login_budget.py` | старый хеш `m=8192,t=1` пересчитан в `m=19456,t=2` при входе; 5 неудач на api-1 и api-2 вместе → 429 даже с верным паролем; тело 429 без деталей | `evidence/auth_acl.txt`, `evidence/login_budget.json` |
| 4: ротация одного рабочего secret без пересборки | `test_secret_rotation.py` (оператор) | пароль `app_worker` изменён в PostgreSQL и OpenBao; оба worker переключились за 10 с; после принудительного переподключения задания обрабатываются; старый пароль → `password authentication failed`; 0 ошибок, 0 упавших заданий | `evidence/secret_rotation.txt` |
| 4: ротация leaf без пересборки, отзыв старой identity | `test_cert_rotation.py` | `lb-client` перевыпущен (serial …1418 → …142A), `nginx -s reload` за 1.4 с; после отзыва старый сертификат закрыт на обеих репликах (`reason=revoked`); 161 запрос, 0 ошибок | `evidence/cert_rotation.txt` |
| 4: ограниченное поведение при expiry и sealed OpenBao | `test_bao_sealed.py` (оператор), `test_cert_rotation.py` | sealed: smoke проходит, сервисы держат секреты, перезапущенный worker ждёт и не стартует без секретов, после unseal здоров через 16 с. Просроченный сертификат с верной identity отклоняется | `evidence/bao_sealed.txt`, `evidence/cert_rotation.txt` |
| 4: bootstrap trust и renewal описаны, нет постоянного «секрета для секретов» | `app/openbao.py`, модель угроз | первичная identity — TLS-ключ сервиса; токен 15 мин, renew/re-login каждые 15 с | `security/THREAT_MODEL.md`, `infra/openbao/README.md` |
| 4: read_only, no-new-privileges, drop capabilities, tmpfs, исключения объяснены | `check_hardening.py` | read-only rootfs у **всех** 15 контейнеров, включая stateful; запись только в volumes и tmpfs (для non-root nginx tmpfs смонтированы с его uid). Исключений не понадобилось | `evidence/hardening.txt` |
| 4: dependency/image scan с разбором применимости | Trivy по образу приложения, nginx, OpenBao | приложение: CRITICAL ОС 9 → 2 (исправлений нет, неприменимы), Python CRITICAL 5 → 0, HIGH 43 → 3 (starlette, неприменимы); nginx и OpenBao обновлены | `evidence/image_scan.txt`, `security/THREAT_MODEL.md` (раздел Scan) |
| 4: ≥ 6 угроз с контролями и воспроизводимыми тестами | модель угроз | 14 угроз T1–T14, у каждой отрицательный тест | `security/THREAT_MODEL.md` |
| 4: upload-worker ограничен по CPU/RAM/времени | `variants_check.py` | дочерний процесс с RLIMIT_AS/CPU и таймаутом; изображение 39.5 Мпикс — пик RSS 337 MiB; лимиты пикселей соблюдены | `evidence/variants_limits.txt` |
| 4: TLS-аудит всей матрицы, ACL, отсутствие утечек | `tls_audit.py`, `test_auth_acl.py`, `test_observability.py` | 135 проверок, 0 провалов (A–K); ACL и приватность — выше | `evidence/tls_audit_final.txt` |
| Внутренняя телеметрия и Grafana работают через mTLS | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, Loki и Tempo выполнены через Grafana | `evidence/dashboard_check.txt` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, OpenBao 0.5 CPU / 256 MiB, nginx 0.25 CPU / 64 MiB. Версии — в таблице ролей.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
### Параметры Argon2id (замер в контейнере API, лимит 0.5 CPU, 10 хешей)
| m, KiB | t | p | p50, мс | max, мс | Пик RSS, MiB |
| --- | --- | --- | --- | --- | --- |
| **19456** | **2** | **1** | **11.3** | 66.6 | 37 |
| 12288 | 3 | 1 | 9.3 | 62.0 | 37 |
| 47104 | 1 | 1 | 16.6 | 69.2 | 83 |
| 65536 | 3 | 1 | 171.6 | 175.5 | 101 |
| 65536 | 3 | 4 | 390.0 | 806.1 | 101 |
**Выбор: m = 19 MiB, t = 2, p = 1** (минимум OWASP).
- При 4 параллельных слотах на реплику это ~80 MiB памяти и ~11 мс CPU на вход. Обе реплики выдерживают ~40 входов/с, оставаясь в лимите 512 MiB.
- Вариант `m = 64 MiB, t = 3` дороже в 15 раз по времени: 4 слота заняли бы ~400 MiB из 512 (риск OOM), а пропускная способность входа упала бы до ~3/с на реплику. Это сделало бы DoS через вход проще.
- Замер — на этом ноутбуке. CFS разрешает кратковременно превышать квоту CPU, поэтому под длительной нагрузкой время растёт (см. ниже).
### Бюджет защиты от DoS через вход (`login_budget.py`, 30 с на фазу)
Модель распределённого атакующего: лимит по IP поднят на время замера, у каждой попытки новый несуществующий username. Параллельно идут обычные запросы (POST сообщения, 5 в секунду).
| Фаза | Попыток входа/с | Латентность 401, p50 / p99 | CPU обеих API, % ядра | RSS API | Обычные запросы: ошибки, p50 / p99 |
| --- | --- | --- | --- | --- | --- |
| без атаки | 0 | — | 10.3 | 97 MiB | 0, 9.5 / 46.4 мс |
| 32 атакующих | 40.4 | 803 / 2397 мс | 89.6 (лимит 100) | 183–197 MiB | 0, 28.5 / 295 мс |
Вывод: атака упирается в лимит CPU, но память ограничена слотами Argon2 (рост на ~100 MiB), а обычный трафик продолжает обслуживаться без ошибок, хотя p99 растёт в 6 раз. Отказов 503 «hash_busy» не было, очередь к слоту укладывалась в 3 с. С одного IP атакующий получает 429 после 107–120 попыток в минуту (`auth_acl.txt`).
### Отказы и ротации
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Ротация leaf `lb-client` под нагрузкой 10 запросов/с | без пересборки и простоя; старый отозван | `pki.sh issue` → `nginx -s reload` (1.4 с) → старый и новый действуют → `revoke-serial` → старый отклонён на api-1 и api-2 через ~1 с, новый 200, просроченный отклонён | 161/161 запросов успешно, smoke после — PASS |
| Ротация пароля `app_worker` под трафиком (5 запросов/с + 1 upload/с) | worker переключается без рестарта, старый пароль недействителен | новая версия KV подхвачена обоими worker за 10 с; сессии `app_worker` принудительно завершены, переподключение с новым паролем; 3 upload → ready | 116 запросов, 20 upload, 0 ошибок, 0 упавших заданий |
| OpenBao запечатан на 58 с | работающие сервисы продолжают, новые ждут, ничего не ослабляется | smoke PASS при запечатанном OpenBao; `openbao_unavailable` в логах; перезапущенный worker-2 в состоянии `starting`, ждёт OpenBao; upload обработан вторым worker | после unseal worker-2 healthy через 16 с |
| Рестарт всех хранилищ, включая OpenBao (hook persistence) | данные и сессии сохраняются | PASS; OpenBao распечатан оператором внутри hook | потерь нет |
| Остановка api-1, затем api-2 (failover) | запись восстанавливается ≤ 10 с, без дублей | PASS ×2: 128 и 130 записей, 0 неудачных попыток | 1.27 и 0.67 с |
| Обновление OpenBao 2.1.0 → 2.6.5 (по scan) | хранилище `file` подхватывается | версия 2.6.5 стартовала на старом volume, данные на месте; после unseal вход сервисов работает | требуется unseal (как после любого рестарта) |
### Что нашли и исправили по ходу проверок
- **Valkey ACL** не разрешал `INCRBY`, хотя valkey-py вызывает её для `INCR`: общий лимит входа молча откатился бы на локальный. Найдено аудитом G.
- **Erlang distribution RabbitMQ** (25672) и **кластерный порт OpenBao 2.6** (8201) были доступны из сети. Привязаны к loopback и отключены соответственно (аудит K).
- **Self-метрики Collector** нельзя было отправить по OTLP с клиентским сертификатом. Переведены на loopback-сбор внутри процесса, plaintext-scrape убран совсем.
- **Self-метрики Collector** приходили с `service.instance.id` = случайный UUID (новый после каждого рестарта), это давало рост кардинальности меток. Задано стабильное значение.
- **У worker не было лимита PIDs** (`check_hardening.py`).
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
**Выдерживается** (проверено): отказ реплики API; рестарт всех хранилищ с volumes; запечатанный или перезапущенный OpenBao (без простоя работающих сервисов); недоступная телеметрия (из lab6); компрометация отдельной identity — её права ограничены (аудит), сертификат LB отзывается.
**Не выдерживается и не заявляется:** потеря Docker-host и компрометация хоста или Docker daemon (root на хосте читает `secrets/`); высокая доступность единственных PostgreSQL, nginx, OpenBao и Collector.
**Учебные упрощения и известные ограничения:**
- Ключи CA, все 3 доли unseal и root token — у одного оператора на хосте стенда (`secrets/`, вне Git, 0600/0700). В production — разные держатели, HSM/KMS, отзыв root token.
- Отзыв по serial проверяет только API. RabbitMQ, OpenBao и Collector отзыв не проверяют; там identity отзывается удалением пользователя или роли.
- Collector не связывает `service.name` с сертификатом. В Prometheus, Loki и Tempo нет прав по identity: любой клиент Telemetry Clients CA и пишет, и читает (CA выдан только collector, grafana, operator).
- Истечение токена OpenBao (15 мин) не выжидалось в тесте. Продление и повторный вход видны в логах (`openbao_login`), истечение сертификата проверено просроченным сертификатом.
- Алерты lab6 (сценарии `test_incidents.py`) в lab7 заново не прогонялись. Проверены трассы, логи, приватность и все запросы dashboard через mTLS. Правила алертов загружены (`promtool check rules`: 13 правил).
- Исчерпание пула БД переводит реплику в not ready (из lab6). Отдельный пул для readiness не реализован.
- `FAULT_INJECTION=1` и `X-Fault` допустимы только на учебном стенде. TCP-healthcheck Loki и Tempo оставляет в их логах записи `TLS handshake error … EOF`.
- Starlette 0.41.3 не обновлялась: применимых уязвимостей не осталось, а обновление требует мажорного перехода FastAPI.
- Первый запуск PKI и OpenBao — ручные шаги оператора; они воспроизводимы скриптами.
**Следующие шаги:** короткоживущие сертификаты из OpenBao PKI с автоматическим renewal (трек 2), проверка отзыва во всех сервисах (CRL/OCSP), отдельный пул для readiness, обновление FastAPI и Starlette.
View File
+81
View File
@@ -0,0 +1,81 @@
"""RabbitMQ topology and confirmed publishing (aio-pika).
Topology (lab7: declared by the operator in infra/rabbitmq/definitions.json; services only check it passively):
exchange attachments direct, durable
queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit
exchange attachments.dlx fanout, durable
queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator)
Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker
has taken responsibility for the message (written to the quorum log).
"""
import json
import logging
import ssl
import aio_pika
from aio_pika import DeliveryMode, ExchangeType, Message
from app import config
log = logging.getLogger("messenger.broker")
ROUTING_KEY = "attachment.process.v1"
def tls_context() -> ssl.SSLContext:
"""lab7: AMQPS with mutual TLS. The broker verifies our client certificate and maps its CN to the RabbitMQ user
(auth=external): api.client.messenger may only publish, worker.client.messenger may only consume."""
ctx = ssl.create_default_context(cafile=config.TLS_CA_FILE)
ctx.load_cert_chain(config.TLS_CLIENT_CERT_FILE, config.TLS_CLIENT_KEY_FILE)
return ctx
async def connect():
return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, ssl_context=tls_context(),
client_properties={"connection_name": config.INSTANCE_ID})
async def get_exchange(channel):
"""Passive: the topology (exchanges, quorum queues, DLX, bindings) is created by the operator from
infra/rabbitmq/definitions.json. Services have NO configure permission, so they only check that it exists."""
return await channel.get_exchange(config.BROKER_EXCHANGE, ensure=True)
async def get_queue(channel):
return await channel.get_queue(config.BROKER_QUEUE, ensure=True)
class Publisher:
def __init__(self) -> None:
self.connection = None
self.channel = None
self.exchange = None
async def ensure(self) -> None:
if self.channel is not None and not self.channel.is_closed:
return
if self.connection is None or self.connection.is_closed:
self.connection = await connect()
self.channel = await self.connection.channel(publisher_confirms=True)
self.exchange = await get_exchange(self.channel)
async def publish(self, body: dict) -> None:
await self.ensure()
message = Message(
json.dumps(body, separators=(",", ":")).encode(),
content_type="application/json",
delivery_mode=DeliveryMode.PERSISTENT,
message_id=body["job_id"],
type=body["type"],
# Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling.
headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}},
)
# mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack.
await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5)
async def close(self) -> None:
try:
if self.connection is not None:
await self.connection.close()
except Exception:
pass
self.connection = self.channel = self.exchange = None
+134
View File
@@ -0,0 +1,134 @@
import os
import socket
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
path = os.environ.get(name + "_FILE")
if path:
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
return os.environ.get(name, default)
# --- lab7: TLS material (paths only) and secrets from OpenBao ------------------------------------------------
TLS_CA_FILE = os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt")
TLS_CLIENT_CERT_FILE = os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt")
TLS_CLIENT_KEY_FILE = os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key")
DATABASE_SSLMODE = os.environ.get("DATABASE_SSLMODE", "verify-full")
BAO_REFRESH_SECONDS = _float("BAO_REFRESH_SECONDS", 15.0)
SECRETS = {}
if os.environ.get("BAO_ADDR"):
from app import openbao as _openbao
SECRETS = _openbao.load(_float("BAO_STARTUP_TIMEOUT_SECONDS", 120.0))
def _from_bao(key, env_name, default=""):
"""OpenBao value when the service runs with BAO_ADDR, otherwise the old *_FILE / env (tools, tests)."""
return SECRETS.get(key) or _secret(env_name, default)
DATABASE_URL = _from_bao("database_url", "DATABASE_URL")
CACHE_URL = _from_bao("cache_url", "CACHE_URL")
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
READY_FAILURES = _int("READY_FAILURES", 2)
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
PASSWORD_MIN = 12
PASSWORD_MAX = 128
SESSION_REVOKED_TTL_SECONDS = max(
_int("SESSION_REVOKED_TTL_SECONDS", 60),
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
)
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = _from_bao("s3_access_key_id", "S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = _from_bao("s3_secret_access_key", "S3_SECRET_ACCESS_KEY")
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
# --- lab5: broker, jobs, worker ---
BROKER_URL = _secret("BROKER_URL") # amqps://...?auth=external: identity = client certificate, no password
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20)
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2)
WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60)
JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20)
JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0)
JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0)
JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0)
WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768)
WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25)
WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0)
OUTBOX_BATCH = _int("OUTBOX_BATCH", 50)
OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400)
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables).
MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200)
PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0)
THUMBNAIL_SIZE = 256
CROP_SIZE = 128
OPTIMIZED_SIZE = 1280
JPEG_QUALITY = _int("JPEG_QUALITY", 85)
WEBP_QUALITY = _int("WEBP_QUALITY", 80)
# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py).
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
# --- lab7: authentication hardening ---------------------------------------------------------------------------
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456)
ARGON2_ITERATIONS = _int("ARGON2_ITERATIONS", 2)
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4)
PASSWORD_HASH_QUEUE_SECONDS = _float("PASSWORD_HASH_QUEUE_SECONDS", 3.0)
LOGIN_RATE_LIMIT_PER_USER = _int("LOGIN_RATE_LIMIT_PER_USER", 5)
LOGIN_RATE_LIMIT_PER_IP = _int("LOGIN_RATE_LIMIT_PER_IP", 120)
LOGIN_RATE_WINDOW_SECONDS = _int("LOGIN_RATE_WINDOW_SECONDS", 60)
# mTLS on the API listener: only these client identities (DNS SAN) may connect; revoked serials are refused.
ALLOWED_CLIENT_IDENTITIES = frozenset(
x.strip() for x in os.environ.get("ALLOWED_CLIENT_IDENTITIES", "lb.client.messenger").split(",") if x.strip())
REVOKED_SERIALS_FILE = os.environ.get("REVOKED_SERIALS_FILE", "")
TLS_SERVER_CERT_FILE = os.environ.get("TLS_SERVER_CERT_FILE", "")
TLS_SERVER_KEY_FILE = os.environ.get("TLS_SERVER_KEY_FILE", "")
HEALTH_LISTEN_PORT = _int("HEALTH_LISTEN_PORT", 8081)
GRACEFUL_TIMEOUT = _int("GRACEFUL_TIMEOUT", 8)
+43
View File
@@ -0,0 +1,43 @@
import base64
import json
import re
import uuid
from datetime import datetime, timezone
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
class BadCursor(Exception):
pass
def fmt(ts: datetime) -> str:
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
def decode_cursor(scope: str, cursor: str):
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
data = json.loads(raw)
except Exception:
raise BadCursor() from None
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
raise BadCursor()
ts, item_id = data["t"], data["i"]
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
raise BadCursor()
try:
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
if str(uuid.UUID(item_id)) != item_id:
raise BadCursor()
except ValueError:
raise BadCursor() from None
if encode_cursor(scope, ts, item_id) != cursor:
raise BadCursor()
return parsed, item_id
+145
View File
@@ -0,0 +1,145 @@
import asyncio
import logging
import re
import time
from contextlib import asynccontextmanager
import psycopg
from opentelemetry.trace import SpanKind, Status, StatusCode
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from app import config, telemetry
from app.errors import unavailable
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)")
TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE)
POOL_WAIT_SPAN_SECONDS = 0.005
log = logging.getLogger("messenger.db")
def statement(query) -> tuple:
"""Span name and attributes for a parametrised SQL statement (values are never part of the text)."""
text = query if isinstance(query, str) else str(query)
op = OPERATION_RE.match(text)
table = TABLE_RE.search(text)
operation = op.group(1).upper() if op else "SQL"
name = operation + (" " + table.group(1) if table else "")
return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger",
"db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]}
async def discard(conn) -> None:
try:
await conn.close()
except Exception:
pass
class Guarded:
def __init__(self, conn, timeout: float):
self.conn = conn
self.timeout = timeout
async def execute(self, *args, **kwargs):
# Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included).
# (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.)
name, attrs = statement(args[0] if args else "")
with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span:
try:
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
except asyncio.TimeoutError:
await discard(self.conn)
span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout))
raise unavailable("Хранилище не ответило за отведённое время") from None
def transaction(self):
return self.conn.transaction()
class Database:
def __init__(self) -> None:
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS * 2,
)
self.pool = AsyncConnectionPool(
conninfo=config.DATABASE_URL,
min_size=config.DB_POOL_MIN,
max_size=config.DB_POOL_MAX,
timeout=config.DB_POOL_TIMEOUT,
max_lifetime=600.0,
open=False,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": config.DB_CONNECT_TIMEOUT,
"options": options,
# lab7: TLS with full server verification (CA + hostname "postgres"); no fallback to plaintext.
"sslmode": config.DATABASE_SSLMODE,
"sslrootcert": config.TLS_CA_FILE,
},
)
def rotate(self, conninfo: str) -> None:
"""New credentials from OpenBao: new connections use them; open ones keep working until max_lifetime."""
if conninfo and conninfo != self.pool.conninfo:
self.pool.conninfo = conninfo
log.info("db_credentials_rotated: new connections use the new password")
async def open(self) -> None:
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close()
@asynccontextmanager
async def connection(self):
started, acquired = time.time_ns(), False
try:
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
acquired = True
waited = (time.time_ns() - started) / 1e9
if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans
telemetry.tracer.start_span("db.pool.acquire", start_time=started,
attributes={"db.system": "postgresql",
"db.client.connection.wait_s": round(waited, 4)}).end()
yield Guarded(conn, config.DB_OP_TIMEOUT)
except BROKEN as exc:
if not acquired:
span = telemetry.tracer.start_span("db.pool.acquire", start_time=started,
attributes={"db.system": "postgresql"})
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
span.end()
raise unavailable("Хранилище недоступно") from None
@asynccontextmanager
async def transaction(self):
async with self.connection() as conn:
async with conn.transaction():
yield conn
async def in_transaction(self, action):
async def run():
async with self.transaction() as conn:
return await action(conn)
try:
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
except asyncio.TimeoutError:
raise unavailable("Хранилище не ответило за отведённое время") from None
async def ready(self) -> bool:
try:
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
try:
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
except asyncio.TimeoutError:
await discard(conn)
return False
return True
except Exception:
return False
+37
View File
@@ -0,0 +1,37 @@
from typing import Optional
from fastapi.responses import JSONResponse
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(scheme: str = "Bearer") -> ApiError:
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found() -> ApiError:
return ApiError(404, "not_found", "Объект не найден")
def unavailable(message: str) -> ApiError:
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": rid}},
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
)
+38
View File
@@ -0,0 +1,38 @@
"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off).
A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`.
Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata.
Crash faults fire only on the first attempt and first delivery, so recovery can be observed.
api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish
worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT
worker_crash_after_commit worker exits after the result COMMIT, before broker ack
worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1
transientN attempts 1..N fail with a transient error (N = 1..9)
permanent permanent error on the first attempt -> failed without retries
"""
import logging
import os
import re
from typing import Optional
from app import config
log = logging.getLogger("messenger.faults")
TRANSIENT_RE = re.compile(r"fault-transient([1-9])")
def active(filename: Optional[str], name: str) -> bool:
return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename
def transient_until(filename: Optional[str]) -> int:
if not config.FAULT_INJECTION or not filename:
return 0
match = TRANSIENT_RE.search(filename)
return int(match.group(1)) if match else 0
def crash(point: str) -> None:
log.error("FAULT INJECTION: process exits at %s", point)
logging.shutdown()
os._exit(70)
+213
View File
@@ -0,0 +1,213 @@
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
only a warning. Both finish before anything is stored.
"""
import logging
import os
import struct
import subprocess
import threading
import time
import zlib
from PIL import Image
from app import config
from app.errors import ApiError, unavailable
log = logging.getLogger("messenger.imagecheck")
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
Image.MAX_IMAGE_PIXELS = None
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
FULL_DECODE_MAX_PIXELS = 4000000
SCAN_SECONDS = 10.0
OUT_STEP = 1 << 20
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
jpeg_slot = threading.Lock()
class Invalid(Exception):
pass
def invalid() -> ApiError:
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
class RowPlan:
def __init__(self, segments):
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
self.seg = 0
self.done = 0
self.pos = 0
def feed(self, data: bytes) -> None:
i, n = 0, len(data)
while i < n:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
if self.seg >= len(self.segments):
raise Invalid("data after the last row")
length, rows = self.segments[self.seg]
if self.pos > 0:
take = min(length - self.pos, n - i)
i += take
self.pos += take
if self.pos == length:
self.pos = 0
self.done += 1
continue
full = min((n - i) // length, rows - self.done)
if full > 0:
if max(data[i:i + full * length:length]) > 4:
raise Invalid("bad filter type")
i += full * length
self.done += full
else:
if data[i] > 4:
raise Invalid("bad filter type")
self.pos = n - i
i = n
def complete(self) -> bool:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
return self.seg >= len(self.segments) and self.pos == 0
def png_plan(width, height, depth, color, interlace):
bits = CHANNELS[color] * depth
if not interlace:
return RowPlan([(1 + (width * bits + 7) // 8, height)])
segments = []
for x0, y0, dx, dy in ADAM7:
w = (width - x0 + dx - 1) // dx if width > x0 else 0
h = (height - y0 + dy - 1) // dy if height > y0 else 0
if w and h:
segments.append((1 + (w * bits + 7) // 8, h))
return RowPlan(segments)
def scan_png(fileobj) -> None:
deadline = time.monotonic() + SCAN_SECONDS
fileobj.seek(0)
if fileobj.read(8) != PNG_SIGNATURE:
raise Invalid("signature")
plan = decomp = None
palette = seen_idat = idat_closed = False
color = None
first = True
while True:
if time.monotonic() > deadline:
raise Invalid("scan too slow")
head = fileobj.read(8)
if len(head) < 8:
raise Invalid("truncated before IEND")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise Invalid("chunk length")
if first and kind != b"IHDR":
raise Invalid("IHDR must be first")
crc = zlib.crc32(kind)
if kind == b"IHDR":
body = fileobj.read(length)
if not first or length != 13 or len(body) != 13:
raise Invalid("IHDR")
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
raise Invalid("IHDR fields")
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
raise Invalid("color type / depth")
plan = png_plan(width, height, depth, color, interlace)
decomp = zlib.decompressobj()
crc = zlib.crc32(body, crc)
else:
remaining = length
while remaining:
buf = fileobj.read(min(remaining, 65536))
if not buf:
raise Invalid("truncated chunk")
remaining -= len(buf)
crc = zlib.crc32(buf, crc)
if kind == b"IDAT":
if idat_closed or (color == 3 and not palette):
raise Invalid("IDAT order")
seen_idat = True
out = decomp.decompress(buf, OUT_STEP)
while True:
plan.feed(out)
if not decomp.unconsumed_tail:
break
if time.monotonic() > deadline:
raise Invalid("scan too slow")
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
if kind == b"PLTE":
palette = True
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise Invalid("PLTE")
elif kind != b"IDAT" and seen_idat:
idat_closed = True
tail = fileobj.read(4)
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
raise Invalid("crc")
first = False
if kind == b"IEND":
if length or not seen_idat:
raise Invalid("IEND")
break
if not decomp.eof or decomp.unused_data or not plan.complete():
raise Invalid("pixel data does not match IHDR")
def check_jpeg(fileobj) -> None:
fileobj.flush()
descriptor = fileobj.fileno()
os.lseek(descriptor, 0, os.SEEK_SET)
try:
with jpeg_slot:
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=SCAN_SECONDS, check=False)
except subprocess.TimeoutExpired:
log.error("jpeg_check_timeout")
raise unavailable("Проверка изображения не успела завершиться") from None
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
if done.returncode < 0:
log.error("jpeg_check_killed signal=%d", -done.returncode)
raise unavailable("Проверка изображения прервана")
if done.returncode != 0:
raise invalid()
def inspect_image(fileobj, size: int, content_type: str) -> None:
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
try:
fileobj.seek(0)
img = Image.open(fileobj)
width, height = img.size
if img.format != FORMATS[content_type]:
raise invalid()
if width * height > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
if content_type == "image/png":
scan_png(fileobj)
if width * height <= FULL_DECODE_MAX_PIXELS:
fileobj.seek(0)
Image.open(fileobj).load()
else:
check_jpeg(fileobj)
except ApiError:
raise
except Exception:
raise invalid() from None
finally:
fileobj.seek(0)
+115
View File
@@ -0,0 +1,115 @@
"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a
wall-clock timeout, so a hostile or huge image can only kill this process, never the worker.
usage: python -m app.imageproc <source> <out_dir> <max_pixels>
stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex.
Rules (contracts/README.md "Фоновая обработка"):
thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px
crop JPEG, centred square scaled to 128x128 (small sources are upscaled)
optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4
EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha).
EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB).
"""
import json
import math
import os
import sys
from PIL import Image, ImageFile, ImageOps
Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding
ImageFile.LOAD_TRUNCATED_IMAGES = False
THUMB, CROP, OPTIMIZED = 256, 128, 1280
JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85"))
WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80"))
ORIENTATION_TAG = 0x0112
EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5
def fit(width: int, height: int, bound: int):
"""Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales."""
if width <= bound and height <= bound:
return width, height
if width >= height:
return bound, max(1, height * bound // width)
return max(1, width * bound // height), bound
def has_alpha(img) -> bool:
return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info)
def to_8bit(img):
if img.mode in ("I;16", "I;16B", "I;16L", "I"):
return img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
return img
def on_white(img):
if has_alpha(img):
rgba = img.convert("RGBA")
flat = Image.new("RGB", rgba.size, (255, 255, 255))
flat.paste(rgba, mask=rgba.getchannel("A"))
return flat, rgba
return img.convert("RGB"), None
def process(source: str, out_dir: str, max_pixels: int) -> list:
img = Image.open(source)
if img.format not in ("PNG", "JPEG"):
raise ValueError("format")
width, height = img.size
if width * height > max_pixels:
sys.exit(EXIT_TOO_LARGE)
orientation = img.getexif().get(ORIENTATION_TAG, 1)
upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height)
thumb_size = fit(*upright, THUMB)
opt_size = fit(*upright, OPTIMIZED)
if img.format == "JPEG":
# DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept).
need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0])
img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1])))
img.load()
img = ImageOps.exif_transpose(img)
img = to_8bit(img)
flat, rgba = on_white(img)
del img
results = []
def save(name, image, ext, fmt, content_type, **params):
path = os.path.join(out_dir, name + "." + ext)
image.save(path, fmt, **params)
results.append({"name": name, "file": path, "content_type": content_type,
"width": image.size[0], "height": image.size[1]})
jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False}
save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0),
"jpg", "JPEG", "image/jpeg", **jpeg)
w, h = flat.size
side = min(w, h)
left, top = (w - side) // 2, (h - side) // 2
save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)),
"jpg", "JPEG", "image/jpeg", **jpeg)
source_for_webp = rgba if rgba is not None else flat
optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize(
opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0)
save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4)
return results
def main() -> None:
source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3])
try:
results = process(source, out_dir, max_pixels)
except MemoryError:
sys.exit(EXIT_TOO_COMPLEX)
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
sys.exit(EXIT_INVALID)
json.dump(results, sys.stdout)
if __name__ == "__main__":
main()
+340
View File
@@ -0,0 +1,340 @@
"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker.
State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it
(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes
`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT.
enqueue nothing -> queued (API, same txn as attachment 'queued')
claim queued(due) -> processing, lease (worker; also takes over an expired lease)
heartbeat processing: lease_until += lease (only while owner is still alive)
complete processing -> succeeded + variants (fenced by lease_owner)
fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts)
sweep_leases processing(expired lease) -> queued | failed
replay failed -> queued (explicit operator action, audited)
"""
import json
import random
import uuid
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from app import config, telemetry
from app.cursor import fmt
JOB_TYPE = "attachment.process.v1"
SCHEMA_VERSION = 1
VARIANT_ORDER = ("thumbnail", "crop", "optimized")
REQUIRED_VARIANTS = VARIANT_ORDER
def backoff_seconds(attempt: int) -> float:
"""Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd)."""
ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)))
return ceiling / 2 + random.uniform(0, ceiling / 2)
def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict:
return {
"schema_version": SCHEMA_VERSION,
"job_id": str(job_id),
"type": JOB_TYPE,
"attachment_id": str(attachment_id),
"pipeline_version": pipeline_version,
"attempt": attempt,
"created_at": fmt(created_at),
}
def parse_message(body: bytes) -> Optional[dict]:
"""Strict check against jobs.schema.json; anything else goes to the DLQ untouched."""
try:
data = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
return None
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}:
return None
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
return None
if "traceparent" in data and not (isinstance(data["traceparent"], str)
and telemetry.TRACEPARENT_RE.match(data["traceparent"])):
return None
if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512):
return None
for key in ("pipeline_version", "attempt"):
if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1:
return None
try:
data["job_id"] = str(uuid.UUID(data["job_id"]))
data["attachment_id"] = str(uuid.UUID(data["attachment_id"]))
except (ValueError, AttributeError, TypeError):
return None
return data
async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None):
await conn.execute(
"INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())",
(job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})),
)
async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0):
body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"])
# W3C trace context of the transaction that schedules the job (upload request span or the failed consumer
# span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate).
body.update(telemetry.inject())
await conn.execute(
"INSERT INTO outbox (job_id, payload, available_at, created_at)"
" VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())",
(job["id"], json.dumps(body), delay),
)
async def enqueue(conn, attachment_id: str, actor: str) -> dict:
cur = await conn.execute(
"INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts,"
" next_attempt_at, created_at, updated_at)"
" VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())"
" RETURNING id, attachment_id, pipeline_version, attempt, created_at",
(attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS),
)
job = await cur.fetchone()
await _outbox(conn, job, 1)
await audit(conn, job["id"], attachment_id, "enqueued", actor, 1)
return job
async def pending_count(conn) -> int:
cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')")
return (await cur.fetchone())["n"]
async def queue_stats(conn) -> dict:
"""Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows."""
cur = await conn.execute(
"SELECT count(*) FILTER (WHERE state = 'queued') AS queued,"
" count(*) FILTER (WHERE state = 'processing') AS processing,"
" coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age"
" FROM attachment_jobs WHERE state IN ('queued', 'processing')"
)
row = dict(await cur.fetchone())
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
row["outbox"] = (await cur.fetchone())["n"]
return row
# ---------------------------------------------------------------- worker side
JOB_COLUMNS = (
"j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until,"
" j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired,"
" (j.next_attempt_at <= clock_timestamp()) AS due"
)
async def claim(conn, msg: dict, owner: str):
"""Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version."""
cur = await conn.execute(
"SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status"
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.id = %s FOR UPDATE OF j",
(msg["job_id"],),
)
job = await cur.fetchone()
if job is None or str(job["attachment_id"]) != msg["attachment_id"]:
return "missing", None
if job["state"] in ("succeeded", "failed"):
return "done", job
if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]:
return "newer_version", job
if msg["attempt"] != job["attempt"]:
return "stale", job
if job["state"] == "processing" and not job["lease_expired"]:
return "busy", job
if job["state"] == "queued" and not job["due"]:
return "early", job
takeover = job["state"] == "processing"
await conn.execute(
"UPDATE attachment_jobs SET state = 'processing', lease_owner = %s,"
" lease_until = clock_timestamp() + make_interval(secs => %s),"
" started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp()),"
" updated_at = clock_timestamp() WHERE id = %s",
(owner, config.JOB_LEASE_SECONDS, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(job["attachment_id"],),
)
await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"])
return "claimed", job
async def heartbeat(conn, job_id, owner: str) -> bool:
cur = await conn.execute(
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
(config.JOB_LEASE_SECONDS, job_id, owner),
)
return cur.rowcount == 1
async def attachment_source(conn, attachment_id) -> Optional[dict]:
cur = await conn.execute(
"SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status"
" FROM attachments WHERE id = %s",
(attachment_id,),
)
return await cur.fetchone()
async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool:
"""Atomically publish the result. False when the lease was lost: someone else owns the effect now."""
cur = await conn.execute(
"UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL,"
" lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
(job["id"], owner),
)
if cur.rowcount != 1:
return False
for v in variants:
await conn.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,"
" size_bytes, width, height, sha256, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())"
" ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
(job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"]),
)
# Never move a newer pipeline_version back, never touch an attachment that is already ready.
await conn.execute(
"UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()"
" WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s",
(job["pipeline_version"], job["attachment_id"], job["pipeline_version"]),
)
await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"],
{"variants": [v["name"] for v in variants]})
return True
async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str:
attempt = job["attempt"]
if transient and attempt < job["max_attempts"]:
delay = backoff_seconds(attempt)
await conn.execute(
"UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL,"
" next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s,"
" updated_at = clock_timestamp() WHERE id = %s",
(delay, detail[:500], error_code, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(job["attachment_id"],),
)
await _outbox(conn, job, attempt + 1, delay=delay)
await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt,
{"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)})
return "retry"
final_code = error_code if not transient else "retry_exhausted"
await conn.execute(
"UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(),"
" last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(detail[:500], final_code, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(final_code, job["attachment_id"]),
)
await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt,
{"error_code": final_code, "cause": error_code, "detail": detail[:200]})
return "failed"
async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str:
"""Returns retry | failed | lost (lease is not ours any more: the new owner decides)."""
cur = await conn.execute(
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE",
(job["id"], owner),
)
row = await cur.fetchone()
if row is None:
return "lost"
return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient)
async def sweep_leases(conn, actor: str) -> List[str]:
"""Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt."""
cur = await conn.execute(
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner"
" FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()"
" ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED",
)
outcomes = []
for row in await cur.fetchall():
outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired",
"lease of %s expired" % (row["lease_owner"] or "?"), True)
outcomes.append("%s:%s" % (row["id"], outcome))
return outcomes
async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict:
"""failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed."""
cur = await conn.execute(
"SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at,"
" a.status AS attachment_status, a.staging_key"
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a",
(attachment_id,),
)
job = await cur.fetchone()
if job is None:
return {"result": "refused", "reason": "no job for attachment"}
if job["state"] != "failed" or job["attachment_status"] != "failed":
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"],
{"state": job["state"], "attachment_status": job["attachment_status"]})
return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])}
if not job["staging_key"] or not await staging_exists(job["staging_key"]):
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"})
return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"}
attempt = job["attempt"] + 1
await conn.execute(
"UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp(),"
" finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s",
(attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'failed'",
(job["attachment_id"],),
)
await _outbox(conn, job, attempt)
await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt)
return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt}
async def failed_jobs(conn, limit: int = 50) -> List[dict]:
cur = await conn.execute(
"SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at"
" FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s",
(limit,),
)
return [
{"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"],
"max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"],
"failed_at": fmt(r["updated_at"])}
for r in await cur.fetchall()
]
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def older_than(seconds: int) -> datetime:
return utcnow() - timedelta(seconds=seconds)
+801
View File
@@ -0,0 +1,801 @@
import asyncio
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import resource
import signal
import tempfile
import unicodedata
import sys
import uuid
from contextlib import asynccontextmanager
from typing import List, Optional, Tuple
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
from starlette.responses import StreamingResponse
from app import config, faults, jobs, openbao, passwords, reconcile, repo, telemetry
from app.cursor import BadCursor, decode_cursor, encode_cursor
from app.db import Database
from app.imagecheck import inspect_image
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
from app.broker import Publisher
from app.outbox import Relay
from app.s3 import S3Storage
from app.sessions import SessionStore
log = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"})
EXEMPT_PATHS = ("/health/live", "/health/ready")
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
JPEG_MAGIC = b"\xff\xd8\xff"
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
background = set()
db = Database()
sessions = SessionStore(db)
s3 = S3Storage()
publisher = Publisher()
relay = Relay(db, publisher)
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0,
"queue": None}
VARIANTS = ("original", "thumbnail", "crop", "optimized")
def rss_mb() -> float:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
except OSError:
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
def count_rejection(code: str) -> None:
state["rejected"][code] = state["rejected"].get(code, 0) + 1
async def sweeper() -> None:
while True:
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
try:
removed = await sessions.sweep()
if removed:
log.info("sessions_expired removed=%d", removed)
except Exception:
log.warning("session_sweep_skipped: storage unavailable")
try:
async with db.connection() as conn:
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
if removed:
log.info("idempotency_cleanup removed=%d", removed)
except Exception:
log.warning("idempotency_cleanup_skipped")
try:
removed = await relay.cleanup()
if removed:
log.info("outbox_cleanup removed=%d", removed)
except Exception:
log.warning("outbox_cleanup_skipped")
async def pending_watch() -> None:
"""Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second."""
while True:
try:
async with db.connection() as conn:
queue = await jobs.queue_stats(conn)
state["pending_jobs"] = queue["queued"] + queue["processing"]
state["queue"] = queue
except Exception:
state["queue"] = None # unknown is not zero: the gauge disappears instead of lying
await asyncio.sleep(config.PENDING_CHECK_SECONDS)
async def reconciler_loop() -> None:
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
while True:
try:
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
log.info("reconcile %s", result)
except Exception:
log.warning("reconcile_skipped: dependency unavailable")
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
async def readiness_watch() -> None:
failures = 0
while True:
if await db.ready():
failures = 0
if not state["ready"]:
state["ready"] = True
log.info("ready_again: database reachable")
else:
failures += 1
if failures >= config.READY_FAILURES and state["ready"]:
state["ready"] = False
log.warning("not_ready: database unreachable, business requests get 503")
s3_ok = await run_in_threadpool(s3.probe)
if s3_ok != state["s3_ok"]:
state["s3_ok"] = s3_ok
log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
await asyncio.sleep(config.READY_CHECK_SECONDS)
@asynccontextmanager
async def lifespan(_app: FastAPI):
previous = {}
def on_signal(signum, frame):
if not state["draining"]:
state["draining"] = True
log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name)
handler = previous.get(signum)
if callable(handler):
handler(signum, frame)
for sig in (signal.SIGTERM, signal.SIGINT):
previous[sig] = signal.getsignal(sig)
signal.signal(sig, on_signal)
await db.open()
telemetry.register_pool(db)
telemetry.register_queue(state)
await sessions.open()
await run_in_threadpool(s3.open)
state["s3_ok"] = await run_in_threadpool(s3.probe)
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()),
asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())]
health_server = await asyncio.start_server(health_listener, "127.0.0.1", config.HEALTH_LISTEN_PORT)
bao = openbao.client()
if bao is not None:
# Rotation without restart: a new KV version swaps DB and S3 credentials for new connections.
bao.on_change(lambda data: db.rotate(data.get("database_url", "")))
bao.on_change(s3.rotate)
bao.start_refresher(config.BAO_REFRESH_SECONDS)
log.info(
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
)
log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s",
config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION)
yield
for task in tasks:
task.cancel()
health_server.close()
await publisher.close()
await sessions.close()
await db.close()
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
telemetry.shutdown()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
@app.middleware("http")
async def guard(request: Request, call_next):
rid = uuid.uuid4().hex
request.state.rid = rid
# X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request.
telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID)
exempt = request.scope["path"] in EXEMPT_PATHS
if state["draining"] and request.scope["path"] != "/health/live":
count_rejection("draining")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
if not exempt and not state["ready"]:
count_rejection("not_ready")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
count_rejection("inflight")
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
state["inflight"] += 1
try:
fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else ""
if fault:
await inject_http_fault(fault)
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
except asyncio.TimeoutError:
count_rejection("request_timeout")
log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.scope["path"])
response = error_body(
rid, config.INSTANCE_ID, 503, "unavailable",
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
)
except Exception as exc:
telemetry.record_exception(exc)
log.exception("unhandled_error request_id=%s", rid)
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
finally:
state["inflight"] -= 1
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
async def inject_http_fault(fault: str) -> None:
"""Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error`
raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose
trace survival can be counted (span attribute messenger.fault)."""
telemetry.set_attributes(messenger__fault=fault[:16])
if fault == "slow":
await asyncio.sleep(0.6)
elif fault == "error":
raise RuntimeError("injected handler failure")
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
if exc.status in (413, 429, 503):
count_rejection(exc.code)
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
def parse_uuid(value: object) -> Optional[str]:
if not isinstance(value, str):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def non_blank(value: object, max_len: int) -> bool:
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
return False
return value.translate(ZERO_WIDTH).strip() != ""
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
declared = request.headers.get("content-length")
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise too_large
raw = bytearray()
async for chunk in request.stream():
raw += chunk
if len(raw) > config.MAX_BODY_BYTES:
raise too_large
try:
body = json.loads(bytes(raw).decode("utf-8"))
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Некорректный JSON") from None
if not isinstance(body, dict):
raise bad("Ожидается JSON-объект")
allowed = set(required) | set(optional)
if set(body) - allowed:
raise bad("Лишние поля в запросе")
if set(required) - set(body):
raise bad("Отсутствуют обязательные поля")
return body
def ensure_capacity() -> None:
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
def bearer_token(request: Request) -> str:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
token = value.strip()
if scheme.lower() != "bearer" or not token:
raise unauthorized()
return token
async def actor(request: Request) -> dict:
user = await sessions.resolve(bearer_token(request))
if user is None:
raise unauthorized()
return user
def basic_credentials(request: Request) -> Tuple[str, str]:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized("Basic")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized("Basic") from None
username, separator, password = decoded.partition(":")
if not separator:
raise unauthorized("Basic")
return username, password
async def member_chat(user: dict, chat_id: str) -> dict:
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
async with db.connection() as conn:
chat = await repo.chat_for_member(conn, cid, user["id"])
if chat is None:
raise not_found()
return chat
def page_params(request: Request) -> Tuple[int, Optional[str]]:
query = request.query_params
limit = 50
if "limit" in query:
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
raise bad("limit должен быть целым 1..100")
limit = int(query["limit"])
cursor = query.get("cursor")
if cursor is not None and cursor == "":
raise bad("Некорректный cursor")
return limit, cursor
def keyset(scope: str, cursor: Optional[str]):
if cursor is None:
return None
try:
return decode_cursor(scope, cursor)
except BadCursor:
raise bad("Некорректный cursor") from None
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
return JSONResponse({"items": items, "next_cursor": next_cursor})
def detect_content_type(head: bytes) -> Optional[str]:
if head[:8] == PNG_MAGIC:
return "image/png"
if head[:3] == JPEG_MAGIC:
return "image/jpeg"
return None
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
raise bad("filename: 1..255 символов, не только пробелы")
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
raise bad("filename содержит недопустимые символы")
hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY)
async def hash_slot(func, *args):
"""Login DoS budget: at most PASSWORD_HASH_CONCURRENCY Argon2 computations per replica at a time (each ~19 MiB,
measured time in REPORT.md); a request that cannot get a slot quickly gets 503 instead of queueing unbounded."""
try:
await asyncio.wait_for(hash_slots.acquire(), config.PASSWORD_HASH_QUEUE_SECONDS)
except asyncio.TimeoutError:
count_rejection("hash_busy")
raise ApiError(503, "unavailable", "Сервер перегружен проверкой паролей, повторите позже",
{"Retry-After": "1"}) from None
try:
return await run_in_threadpool(func, *args)
finally:
hash_slots.release()
def client_ip(request: Request) -> str:
"""X-Real-IP is set by nginx. It can be trusted: only the LB identity passes the API mTLS check, so a client
cannot reach the API directly and forge it. Fallback: the TCP peer."""
value = request.headers.get("x-real-ip", "")
return value if re.fullmatch(r"[0-9a-fA-F:.]{2,45}", value) else (request.client.host if request.client else "-")
def rate_limited(scope: str, retry_after: int) -> None:
count_rejection("login_rate_" + scope)
log.info("login_rate_limited scope=%s retry_after=%d", scope, retry_after)
raise ApiError(429, "rate_limited", "Слишком много попыток входа, повторите позже",
{"Retry-After": str(int(retry_after))})
@app.get("/health/live")
async def live():
return {"status": "ok"}
async def health_listener(reader, writer) -> None:
"""Loopback-only (127.0.0.1) plain HTTP for the Docker healthcheck: /health/live and /health/ready only.
It is not published and not reachable from the network, so it is no bypass of the mTLS business listener."""
try:
line = (await asyncio.wait_for(reader.readline(), 2)).decode("latin-1")
path = line.split(" ")[1] if line.count(" ") >= 2 else ""
if path == "/health/live":
status = 200
elif path == "/health/ready":
status = 200 if (not state["draining"] and state["s3_ok"] and await db.ready()) else 503
else:
status = 404
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
writer.write(b"HTTP/1.1 %d X\r\nContent-Type: application/json\r\nContent-Length: %d\r\n"
b"Connection: close\r\n\r\n%s" % (status, len(body), body))
await writer.drain()
except Exception:
pass
finally:
writer.close()
@app.get("/health/ready")
async def ready():
if not await db.ready():
raise unavailable("База данных недоступна")
if not state["s3_ok"]:
raise unavailable("Хранилище файлов недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
body = await read_json(request, ("username", "display_name", "password"))
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
if not non_blank(body["display_name"], 100):
raise bad("display_name: 1..100 символов, не только пробелы")
password = body["password"]
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
ensure_capacity()
password_hash = await hash_slot(passwords.hash_password, password)
async with db.connection() as conn:
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
if user is None:
raise ApiError(409, "conflict", "username уже занят")
log.info("user_created id=%s", user["id"])
return JSONResponse(user, status_code=201)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
if await request.body():
raise bad("POST /auth/sessions не принимает тело запроса")
username, password = basic_credentials(request)
row = None
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
async with db.connection() as conn:
row = await repo.credentials(conn, username)
# lab7: limits BEFORE the expensive Argon2 verification; counters are shared by all replicas (Valkey).
ip_key = "ratelimit:login:ip:" + client_ip(request)
user_key = "ratelimit:login:user:" + (username if USERNAME_RE.fullmatch(username) else "-invalid-")
count, retry = await sessions.hit(ip_key, config.LOGIN_RATE_WINDOW_SECONDS)
if count > config.LOGIN_RATE_LIMIT_PER_IP:
rate_limited("ip", retry)
failures, retry = await sessions.peek(user_key)
if failures >= config.LOGIN_RATE_LIMIT_PER_USER:
rate_limited("user", retry)
stored = row["password_hash"] if row else passwords.DUMMY_HASH
matches = await hash_slot(passwords.verify_password, stored, password)
if row is None or not matches:
await sessions.hit(user_key, config.LOGIN_RATE_WINDOW_SECONDS)
log.info("login_failed username_known=%s", row is not None)
raise unauthorized("Basic")
await sessions.reset(user_key)
if passwords.needs_rehash(stored):
# Old parameters (or another algorithm) -> re-hash with the current Argon2id parameters while we know the
# plaintext; a concurrent login doing the same is harmless (both hashes are valid).
new_hash = await hash_slot(passwords.hash_password, password)
async with db.connection() as conn:
await repo.update_password_hash(conn, row["id"], new_hash)
log.info("password_rehashed user=%s", row["id"])
user = repo.public_user(row)
token, expires_at = await sessions.create(user)
log.info("session_created user=%s", user["id"])
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request):
token = bearer_token(request)
user = await sessions.resolve(token)
if user is None:
raise unauthorized()
await sessions.revoke(token)
log.info("session_revoked user=%s", user["id"])
return Response(status_code=204)
@app.get("/api/v1/users/me")
async def me(request: Request):
return await actor(request)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await actor(request)
body = await read_json(request, ("title", "member_ids"))
if not non_blank(body["title"], 100):
raise bad("title: 1..100 символов, не только пробелы")
raw_ids = body["member_ids"]
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
raise bad("member_ids: массив из 1..100 UUID")
ids: List[str] = []
for value in raw_ids:
parsed = parse_uuid(value)
if parsed is None:
raise bad("member_ids содержит некорректный UUID")
ids.append(parsed)
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user["id"] not in ids:
raise bad("Создатель должен входить в member_ids")
ensure_capacity()
async def insert(conn):
return await repo.create_chat(conn, body["title"], ids)
try:
chat = await db.in_transaction(insert)
except repo.UnknownMember:
raise bad("member_ids содержит неизвестного пользователя") from None
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
return JSONResponse(chat, status_code=201)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = page_params(request)
scope = "chats:" + user["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
return page(scope, items, has_more)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, request: Request):
return await member_chat(await actor(request), chat_id)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
user = await actor(request)
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
idem_key = request.headers.get("idempotency-key", "")
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
body = await read_json(request, ("text",), ("attachment_ids",))
text = body["text"]
attachment_ids_raw = body.get("attachment_ids", [])
if not isinstance(attachment_ids_raw, list):
raise bad("attachment_ids должен быть массивом UUID")
if len(attachment_ids_raw) > 10:
raise bad("attachment_ids: максимум 10 вложений")
attachment_ids: List[str] = []
for value in attachment_ids_raw:
parsed = parse_uuid(value)
if parsed is None:
raise bad("attachment_ids содержит некорректный UUID")
attachment_ids.append(parsed)
if len(set(attachment_ids)) != len(attachment_ids):
raise bad("attachment_ids содержит дубликаты")
has_text = isinstance(text, str) and non_blank(text, 4000)
has_attachments = len(attachment_ids) > 0
if not has_text and not has_attachments:
raise bad("text: 1..4000 символов, не только пробелы")
if isinstance(text, str) and len(text) > 4000:
raise bad("text: 1..4000 символов, не только пробелы")
if not isinstance(text, str):
raise bad("text должен быть строкой")
ensure_capacity()
normalized = {"text": text}
if attachment_ids:
normalized["attachment_ids"] = attachment_ids
body_hash = hashlib.sha256(
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
async def do(conn):
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
status, message = await db.in_transaction(do)
if status == "conflict":
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
if status == "not_member":
raise not_found()
if status == "attachment_not_found":
raise not_found()
if status == "attachment_failed":
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
return JSONResponse(message, status_code=201)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request):
chat = await member_chat(await actor(request), chat_id)
limit, cursor = page_params(request)
scope = "messages:" + chat["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
return page(scope, items, has_more)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request):
user = await actor(request)
chat = await member_chat(user, chat_id)
filename = request.query_params.get("filename", "")
validate_filename(filename)
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared_ct not in ALLOWED_CONTENT_TYPES:
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
declared_length = request.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
raise too_large
ensure_capacity()
# Backpressure: do not accept more work than the pipeline can hold; the client retries later.
if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS:
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
sha = hashlib.sha256()
size = 0
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
head = b""
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
if len(head) < 8:
head += chunk[: 8 - len(head)]
if len(head) == 8 and detect_content_type(head) is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
sha.update(chunk)
spool.write(chunk)
if size == 0:
raise bad("Пустой файл")
actual_ct = detect_content_type(head)
if actual_ct is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
if actual_ct != declared_ct:
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
async with upload_slots:
await run_in_threadpool(inspect_image, spool, size, actual_ct)
attachment_id = str(uuid.uuid4())
staging_key = "staging/" + attachment_id
object_key = "attachments/%s/original" % attachment_id
async with db.connection() as conn:
await repo.begin_attachment(
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(),
staging_key, object_key,
)
try:
async with upload_slots:
await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct)
except BaseException:
defer(compensate(attachment_id, staging_key))
raise
# Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox.
try:
att = await db.in_transaction(lambda conn: repo.queue_attachment(
conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS))
except BaseException:
await compensate(attachment_id, staging_key)
raise
if att == repo.QUEUE_FULL:
defer(compensate(attachment_id, staging_key))
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
if att is None:
raise unavailable("Загрузка отменена сверкой, повторите запрос")
telemetry.set_attributes(messenger__attachment_id=att["id"])
if faults.active(filename, "api_crash_after_commit"):
faults.crash("API after COMMIT of job/outbox, before publish")
relay.notify()
log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size)
return JSONResponse(att, status_code=202)
def defer(coro) -> None:
task = asyncio.ensure_future(coro)
background.add(task)
task.add_done_callback(background.discard)
async def compensate(attachment_id: str, object_key: str) -> None:
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
try:
async with db.connection() as conn:
await repo.abort_attachment(conn, attachment_id)
await run_in_threadpool(s3.delete, object_key)
except BaseException:
log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id)
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
user = await actor(request)
aid = parse_uuid(attachment_id)
if aid is None:
raise bad("attachment_id должен быть UUID")
async with db.connection() as conn:
row = await repo.get_attachment(conn, aid)
if row is None:
raise not_found()
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
if chat is None:
raise not_found()
return row
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variants = []
if row["status"] == "ready":
async with db.connection() as conn:
variants = await repo.get_variants(conn, row["id"], row["pipeline_version"])
return repo._attachment(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variant = request.query_params.get("variant", "original")
if variant not in VARIANTS:
raise bad("variant: original, thumbnail, crop или optimized")
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing")
else "Обработка вложения завершилась ошибкой")
key, content_type = row["object_key"], row["content_type"]
if variant != "original":
async with db.connection() as conn:
found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant]
if not found:
raise not_found()
key, content_type = found[0]["object_key"], found[0]["content_type"]
opened = await run_in_threadpool(s3.open_stream, key)
if opened is None:
log.error("object_missing attachment=%s variant=%s", row["id"], variant)
raise unavailable("Файл временно недоступен")
chunks, length = opened
return StreamingResponse(
iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)},
)
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def fallback(request: Request):
raise not_found()
# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel
# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics.
telemetry.setup("messenger-api", app)
+161
View File
@@ -0,0 +1,161 @@
"""OpenBao client for the services (lab7): cert auth -> short-lived token -> read ONE scoped KV v2 secret.
Bootstrap trust: the only material a service gets from the operator is its TLS client certificate (Docker secret).
It is also its identity for RabbitMQ (EXTERNAL) and LB->API mTLS is separate. There is no "secret to get secrets"
in the image or in .env: no token, no AppRole SecretID.
login POST /v1/auth/cert/login (client certificate, role = BAO_AUTH_ROLE) -> token (TTL 15 min, max 1 h)
read GET /v1/messenger/data/<BAO_KV_PATH> (policy messenger-<role>: read own path only)
refresh every BAO_REFRESH_SECONDS: renew-self (or re-login near max TTL), re-read metadata; a new KV version
(rotation) is delivered to registered callbacks without a restart
sealed / unreachable OpenBao: the process keeps the secrets it already holds (bounded: existing DB/S3/broker
sessions continue); a NEW process waits up to BAO_STARTUP_TIMEOUT_SECONDS and then exits non-zero.
Secret values are never logged; only paths, versions, TTLs and error classes.
"""
import json
import logging
import os
import ssl
import threading
import time
import urllib.error
import urllib.request
from typing import Callable, Dict, List, Optional
log = logging.getLogger("messenger.openbao")
class OpenBaoError(Exception):
pass
class OpenBao:
def __init__(self) -> None:
self.addr = os.environ["BAO_ADDR"].rstrip("/")
self.role = os.environ["BAO_AUTH_ROLE"]
self.path = os.environ["BAO_KV_PATH"]
self.mount = os.environ.get("BAO_KV_MOUNT", "messenger")
self.ctx = ssl.create_default_context(cafile=os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt"))
self.ctx.load_cert_chain(os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt"),
os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key"))
self.token: Optional[str] = None
self.token_expires = 0.0
self.data: Dict[str, str] = {}
self.version = 0
self.callbacks: List[Callable[[Dict[str, str]], None]] = []
self.state = {"available": False, "last_ok": 0.0, "errors": 0, "logins": 0, "renewals": 0}
self.lock = threading.Lock()
# ------------------------------------------------------------------ HTTP
def _call(self, method: str, path: str, body: Optional[dict] = None, token: Optional[str] = None) -> dict:
req = urllib.request.Request(self.addr + path, method=method,
data=json.dumps(body).encode() if body is not None else None)
req.add_header("Content-Type", "application/json")
if token:
req.add_header("X-Vault-Token", token)
try:
with urllib.request.urlopen(req, context=self.ctx, timeout=5) as response:
raw = response.read()
except urllib.error.HTTPError as exc:
# 503 = sealed / standby, 403 = denied or expired token, 400 = cert not accepted for the role
raise OpenBaoError("HTTP %d on %s" % (exc.code, path.split("?")[0])) from None
except (urllib.error.URLError, OSError) as exc:
raise OpenBaoError("%s on %s" % (type(getattr(exc, "reason", exc)).__name__, path)) from None
return json.loads(raw) if raw else {}
def login(self) -> None:
auth = self._call("POST", "/v1/auth/cert/login", {"name": self.role})["auth"]
now = time.time()
self.token = auth["client_token"]
self.token_expires = now + auth["lease_duration"]
self.state["logins"] += 1
log.info("openbao_login role=%s policies=%s ttl=%ds", self.role, ",".join(auth.get("policies", [])),
auth["lease_duration"])
def _ensure_token(self, interval: float) -> None:
now = time.time()
if self.token is None or now > self.token_expires - 1:
self.login()
return
if self.token_expires - now < 2 * interval + 5:
try:
auth = self._call("POST", "/v1/auth/token/renew-self", {}, self.token)["auth"]
self.token_expires = time.time() + auth["lease_duration"]
self.state["renewals"] += 1
if auth["lease_duration"] < 2 * interval + 5: # reached max TTL: authenticate again
self.login()
except OpenBaoError:
self.login()
def _read(self) -> None:
body = self._call("GET", "/v1/%s/data/%s" % (self.mount, self.path), token=self.token)["data"]
version = int(body["metadata"]["version"])
changed = version != self.version
with self.lock:
self.data, self.version = dict(body["data"]), version
self.state.update(available=True, last_ok=time.time())
if changed:
log.info("openbao_secret_loaded path=%s/%s version=%d keys=%s", self.mount, self.path, version,
",".join(sorted(self.data)))
for callback in list(self.callbacks):
try:
callback(dict(self.data))
except Exception:
log.exception("openbao_rotation_callback_failed")
# ------------------------------------------------------------------ lifecycle
def load_blocking(self, timeout: float) -> Dict[str, str]:
deadline = time.monotonic() + timeout
delay = 1.0
while True:
try:
self.login()
self._read()
return dict(self.data)
except (OpenBaoError, KeyError, ValueError) as exc:
self.state["errors"] += 1
if time.monotonic() > deadline:
log.error("openbao_unavailable_at_start role=%s error=%s: giving up", self.role, exc)
raise SystemExit(78)
log.warning("openbao_not_ready role=%s error=%s (sealed or down?), retry in %.0fs", self.role, exc, delay)
time.sleep(delay)
delay = min(delay * 2, 10.0)
def on_change(self, callback: Callable[[Dict[str, str]], None]) -> None:
self.callbacks.append(callback)
def start_refresher(self, interval: float) -> None:
def loop():
while True:
time.sleep(interval)
try:
self._ensure_token(interval)
meta = self._call("GET", "/v1/%s/metadata/%s" % (self.mount, self.path), token=self.token)
if int(meta["data"]["current_version"]) != self.version:
self._read()
else:
self.state.update(available=True, last_ok=time.time())
except (OpenBaoError, KeyError, ValueError) as exc:
self.state["errors"] += 1
if self.state["available"]:
log.warning("openbao_unavailable role=%s error=%s: keeping cached secret version %d",
self.role, exc, self.version)
self.state["available"] = False
if "403" in str(exc):
self.token = None # expired/revoked token: log in again next round
threading.Thread(target=loop, name="openbao-refresh", daemon=True).start()
_client: Optional[OpenBao] = None
def client() -> Optional[OpenBao]:
return _client
def load(timeout: float) -> Dict[str, str]:
"""Called once by app.config at import time when BAO_ADDR is set."""
global _client
_client = OpenBao()
return _client.load_blocking(timeout)
+100
View File
@@ -0,0 +1,100 @@
"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work).
Rows are locked while they are published; `published_at` is set in the same transaction only after the
broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are
expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish
leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS.
"""
import asyncio
import logging
from opentelemetry.trace import SpanKind
from app import config, telemetry
log = logging.getLogger("messenger.outbox")
class Relay:
def __init__(self, db, publisher) -> None:
self.db = db
self.publisher = publisher
self.wake = asyncio.Event()
self.broker_ok = False
self.published = 0
def notify(self) -> None:
self.wake.set()
async def run_once(self) -> int:
sent, failure = 0, None
async with self.db.connection() as conn:
async with conn.transaction():
cur = await conn.execute(
"SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()"
" ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED",
(config.OUTBOX_BATCH,),
)
for row in await cur.fetchall():
try:
await self.publish(row["payload"])
except Exception as exc:
# Keep what was already confirmed: commit those rows, record the error, stop the batch.
await conn.execute(
"UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s",
(type(exc).__name__, row["id"]),
)
failure = exc
break
await conn.execute(
"UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1"
" WHERE id = %s",
(row["id"],),
)
sent += 1
self.published += sent
if failure is not None:
raise failure
return sent
async def publish(self, payload: dict) -> None:
"""PRODUCER span, child of the context stored with the job (upload request or failed attempt);
the message then carries the producer's own traceparent, so the consumer becomes its child."""
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish",
"messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"],
"messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]}
with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER,
context=telemetry.extract(payload), attributes=attrs):
body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")}
body.update(telemetry.inject())
await self.publisher.publish(body)
async def loop(self) -> None:
delay = config.OUTBOX_POLL_SECONDS
while True:
try:
await asyncio.wait_for(self.wake.wait(), delay)
except asyncio.TimeoutError:
pass
self.wake.clear()
try:
while await self.run_once() == config.OUTBOX_BATCH:
pass
if not self.broker_ok:
log.info("outbox_relay_resumed: broker reachable")
self.broker_ok = True
delay = config.OUTBOX_POLL_SECONDS
except Exception as exc:
if self.broker_ok:
log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__)
self.broker_ok = False
await self.publisher.close()
delay = min(10.0, max(delay * 2, 1.0))
async def cleanup(self) -> int:
async with self.db.connection() as conn:
cur = await conn.execute(
"DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)",
(config.OUTBOX_RETENTION_SECONDS,),
)
return cur.rowcount
+29
View File
@@ -0,0 +1,29 @@
"""Argon2id password storage (lab7). Parameters come from the environment and were measured on the stand
(student-tests/argon2_bench.py, REPORT.md); the PHC string stores salt + parameters, so old hashes stay valid and
are upgraded on the next successful login (needs_rehash)."""
from argon2 import PasswordHasher, Type
from argon2.exceptions import Argon2Error, InvalidHashError, VerifyMismatchError
from app import config
HASHER = PasswordHasher(time_cost=config.ARGON2_ITERATIONS, memory_cost=config.ARGON2_MEMORY_KIB,
parallelism=config.ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID)
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
def hash_password(password: str) -> str:
return HASHER.hash(password)
def verify_password(stored: str, password: str) -> bool:
try:
return HASHER.verify(stored, password)
except (VerifyMismatchError, Argon2Error, InvalidHashError, ValueError):
return False
def needs_rehash(stored: str) -> bool:
try:
return HASHER.check_needs_rehash(stored)
except (InvalidHashError, ValueError):
return True
+155
View File
@@ -0,0 +1,155 @@
"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout).
Key layout (all keys are generated by the server):
staging/<attachment_id> written by the API before 202, read by the worker
attachments/<attachment_id>/original server-side copy made by the worker
attachments/<attachment_id>/v<N>/<variant>.* variants of pipeline version N
Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row.
Cases (all gated by a grace period so in-flight work is never touched):
stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both
orphan_object object without any row/variant row, older than grace -> delete object
staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete
staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete
(after that a replay is refused; the attachment stays failed and visible)
missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing
Queued/processing work is never touched: its staging object is the input of a live job.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
from starlette.concurrency import run_in_threadpool
from app import config
log = logging.getLogger("messenger.reconcile")
LOCK_ID = 2026_0104
PREFIXES = ("staging/", "attachments/")
BATCH = 500
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
started = time.monotonic()
result = {
"dry_run": dry_run, "grace_seconds": grace_seconds,
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0,
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
}
async with db.connection() as conn:
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
if not (await cur.fetchone())["got"]:
return {"skipped": "another reconciler is running"}
try:
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace_seconds)
await _stale_uploading(conn, s3, cutoff, dry_run, result)
await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result)
await _orphan_objects(conn, s3, cutoff, dry_run, result)
await _missing_objects(conn, s3, cutoff, dry_run, result)
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
result["seconds"] = round(time.monotonic() - started, 3)
return result
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, coalesce(staging_key, object_key) AS key FROM attachments"
" WHERE status = 'uploading' AND created_at < %s",
(cutoff,),
)
for row in await cur.fetchall():
modified = await run_in_threadpool(s3.head, row["key"])
if modified is not None and modified >= cutoff:
result["protected_fresh_objects"] += 1
continue
result["stale_uploading"] += 1
if dry_run:
continue
if modified is not None:
await run_in_threadpool(s3.delete, row["key"])
result["deleted_objects"] += 1
done = await conn.execute(
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
)
result["removed_rows"] += done.rowcount
async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL"
" AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
(retention_cutoff,),
)
for row in await cur.fetchall():
if await run_in_threadpool(s3.head, row["staging_key"]) is None:
continue
result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1
if dry_run:
continue
again = await conn.execute(
"SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
(row["id"], retention_cutoff),
)
if await again.fetchone() is not None:
await run_in_threadpool(s3.delete, row["staging_key"])
result["deleted_objects"] += 1
async def _known_keys(conn, keys):
cur = await conn.execute(
"SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)"
" UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)"
" UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)",
(keys, keys, keys),
)
return {row["k"] for row in await cur.fetchall()}
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
for prefix in PREFIXES:
pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix)))
while True:
page = await run_in_threadpool(next, pages, None)
if page is None:
break
result["objects_scanned"] += len(page)
keys = [o["Key"] for o in page]
if not keys:
continue
known = await _known_keys(conn, keys)
for obj in page:
if obj["Key"] in known or obj["LastModified"] >= cutoff:
continue
result["orphan_objects"] += 1
if not dry_run and not await _known_keys(conn, [obj["Key"]]):
await run_in_threadpool(s3.delete, obj["Key"])
result["deleted_objects"] += 1
async def _missing_objects(conn, s3, cutoff, dry_run, result):
last = None
while True:
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
(cutoff, last, last, BATCH),
)
rows = await cur.fetchall()
if not rows:
return
for row in rows:
result["rows_scanned"] += 1
if await run_in_threadpool(s3.exists, row["object_key"]):
continue
result["missing_objects"] += 1
if not dry_run:
done = await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
)
result["marked_failed"] += done.rowcount
last = str(rows[-1]["id"])
+342
View File
@@ -0,0 +1,342 @@
import hashlib
import json
from typing import List, Optional, Tuple
import psycopg
from app import config, jobs
from app.cursor import fmt
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
CHAT_QUERY = (
"SELECT c.id, c.title, c.created_at,"
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
)
class UnknownMember(Exception):
pass
def decoded(value) -> str:
return bytes(value).decode("utf-8")
def public_user(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": decoded(row["display_name"]),
"created_at": fmt(row["created_at"]),
}
def _chat(row) -> dict:
return {
"id": str(row["id"]),
"title": decoded(row["title"]),
"member_ids": [str(value) for value in row["member_ids"]],
"created_at": fmt(row["created_at"]),
}
def _message(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": decoded(row["text"]),
"created_at": fmt(row["created_at"]),
}
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
msg = _message(row)
msg["attachment_ids"] = attachment_ids
return msg
VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2}
def _variant(row) -> dict:
return {
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
def _attachment(row, variants=None) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [],
"error_code": row["error_code"],
"created_at": fmt(row["created_at"]),
"updated_at": fmt(row["updated_at"]),
}
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
try:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
" RETURNING id, username, display_name, created_at",
(username, display_name.encode("utf-8"), password_hash),
)
except psycopg.errors.UniqueViolation:
return None
return public_user(await cur.fetchone())
async def update_password_hash(conn, user_id, password_hash: str) -> None:
await conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (password_hash, user_id))
async def credentials(conn, username: str):
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,),
)
return await cur.fetchone()
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_at)"
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
" RETURNING id, title, created_at",
(title.encode("utf-8"),),
)
chat = await cur.fetchone()
cur = await conn.execute(
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
(chat["id"], member_ids),
)
if cur.rowcount != len(member_ids):
raise UnknownMember()
chat["member_ids"] = list(member_ids)
return _chat(chat)
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
cur = await conn.execute(
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id),
)
row = await cur.fetchone()
return _chat(row) if row else None
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
CHAT_QUERY
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
" ORDER BY c.created_at, c.id LIMIT %s",
(user_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
return [_chat(row) for row in rows[:limit]], len(rows) > limit
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
)
row = await cur.fetchone()
return _message(row) if row else None
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
cur = await conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
)
rows = await cur.fetchall()
return [str(row["attachment_id"]) for row in rows]
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
for pos, aid in enumerate(attachment_ids):
await conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, aid, pos),
)
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
" ORDER BY created_at, id LIMIT %s",
(chat_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
messages = []
for row in rows[:limit]:
aids = await _fetch_message_attachment_ids(conn, row["id"])
messages.append(_message_with_attachments(row, aids))
return messages, len(rows) > limit
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
raw = "%s:%s:%s" % (user_id, chat_id, key)
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
attachment_ids: List[str], idem_key: str, body_hash: str):
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
)
if await cur.fetchone() is None:
return "not_member", None
cur = await conn.execute(
"SELECT request_body_hash, response_json FROM idempotency_keys"
" WHERE user_id = %s AND chat_id = %s AND key = %s",
(sender_id, chat_id, idem_key),
)
existing = await cur.fetchone()
if existing:
if existing["request_body_hash"] != body_hash:
return "conflict", None
return "replay", json.loads(existing["response_json"])
if attachment_ids:
cur = await conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
(attachment_ids,),
)
att_rows = await cur.fetchall()
found_ids = {str(row["id"]) for row in att_rows}
for aid in attachment_ids:
if aid not in found_ids:
return "attachment_not_found", None
for row in att_rows:
if str(row["chat_id"]) != chat_id:
return "attachment_not_found", None
if row["status"] == "failed":
return "attachment_failed", None
text_bytes = text.encode("utf-8") if text else b""
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text_bytes, chat_id, sender_id),
)
row = await cur.fetchone()
if row is None:
return "not_member", None
if attachment_ids:
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
message = _message_with_attachments(row, list(attachment_ids))
await conn.execute(
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
)
return "created", message
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
(ttl_seconds,),
)
return cur.rowcount
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" staging_key, status, error_code, pipeline_version, created_at, updated_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())",
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key,
config.PIPELINE_VERSION),
)
ATTACHMENT_COLUMNS = (
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status,"
" error_code, pipeline_version, created_at, updated_at"
)
QUEUE_FULL = "queue_full"
ADMISSION_LOCK = 2026_0105
async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0):
"""uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff).
With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock,
so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight.
"""
if max_pending:
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,))
if await jobs.pending_count(conn) >= max_pending:
return QUEUE_FULL
cur = await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS,
(attachment_id,),
)
row = await cur.fetchone()
if row is None:
return None
await jobs.enqueue(conn, attachment_id, actor)
return _attachment(row)
async def abort_attachment(conn, attachment_id: str) -> None:
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
async def get_attachment(conn, attachment_id: str):
cur = await conn.execute(
"SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'",
(attachment_id,),
)
return await cur.fetchone()
async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]:
cur = await conn.execute(
"SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants"
" WHERE attachment_id = %s AND pipeline_version = %s",
(attachment_id, pipeline_version),
)
rows = await cur.fetchall()
return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99))
+132
View File
@@ -0,0 +1,132 @@
import logging
from typing import Iterator, Optional, Tuple
import boto3
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.errors import unavailable
log = logging.getLogger("messenger.s3")
MISSING = ("NoSuchKey", "404", "NotFound")
class S3Storage:
def __init__(self) -> None:
self.client = None
self.probe_client = None
def open(self, access_key: str = "", secret_key: str = "") -> None:
self.access_key = access_key or config.S3_ACCESS_KEY_ID
self.secret_key = secret_key or config.S3_SECRET_ACCESS_KEY
self.probe_client = self._client(1.0, 1.5)
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
def rotate(self, secrets: dict) -> None:
"""New S3 credentials from OpenBao: swap the clients (boto3 clients are immutable)."""
key, secret = secrets.get("s3_access_key_id"), secrets.get("s3_secret_access_key")
if key and secret and (key, secret) != (self.access_key, self.secret_key):
self.open(key, secret)
log.info("s3_credentials_rotated access_key=%s", key)
def _client(self, connect: float, read: float):
return boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
verify=config.TLS_CA_FILE, # lab7: HTTPS to MinIO, server certificate verified against the lab CA
config=BotoConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"max_attempts": 0},
),
)
def probe(self) -> bool:
try:
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
try:
fileobj.seek(0)
self.client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
)
except (BotoCoreError, ClientError) as exc:
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
except BotoCoreError as exc:
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
body = response["Body"]
return body.iter_chunks(65536), int(response["ContentLength"])
def head(self, key: str):
"""LastModified of the object, or None when it does not exist."""
try:
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
raise
def exists(self, key: str) -> bool:
return self.head(key) is not None
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
def list_pages(self, prefix: str):
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
yield page.get("Contents", [])
# ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient)
def put_file(self, key: str, path: str, content_type: str) -> None:
with open(path, "rb") as handle:
self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type)
def download(self, key: str, path: str) -> bool:
"""False when the object does not exist."""
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return False
raise
with open(path, "wb") as handle:
for chunk in response["Body"].iter_chunks(1 << 20):
handle.write(chunk)
return True
def copy(self, source_key: str, target_key: str) -> None:
"""Server-side copy: the original bytes never pass through the worker again."""
self.client.copy_object(
Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key},
)
def ready(self) -> bool:
try:
self.client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
+95
View File
@@ -0,0 +1,95 @@
"""API process entrypoint (lab7): uvicorn behind MANDATORY mutual TLS with an identity allowlist.
TLS layer (ssl.SSLContext, CERT_REQUIRED): the client must present a certificate chained to the lab root CA.
Identity layer (this module, right after the handshake, for every connection including resumed TLS sessions):
* DNS SAN must be in ALLOWED_CLIENT_IDENTITIES (only the LB: lb.client.messenger);
* the certificate serial must not be in REVOKED_SERIALS_FILE (re-read when it changes, no restart needed).
A certificate from the same CA with another identity (worker, api, intruder) is closed before any HTTP byte is read.
There is no plaintext listener on the network; the Docker healthcheck uses 127.0.0.1:8081 (app.main).
"""
import logging
import os
import ssl
import time
import uvicorn
from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol
from app import config, telemetry
log = logging.getLogger("messenger.mtls")
class Revocations:
def __init__(self, path: str) -> None:
self.path, self.mtime, self.checked, self.serials = path, None, 0.0, frozenset()
def current(self) -> frozenset:
now = time.monotonic()
if self.path and now - self.checked > 1.0:
self.checked = now
try:
mtime = os.stat(self.path).st_mtime
if mtime != self.mtime:
with open(self.path, encoding="utf-8") as handle:
self.serials = frozenset(line.strip().upper() for line in handle if line.strip())
self.mtime = mtime
log.info("mtls_revocations_loaded count=%d", len(self.serials))
except OSError:
pass # keep the last known list
return self.serials
REVOKED = Revocations(config.REVOKED_SERIALS_FILE)
rejected = telemetry.meter.create_counter("messenger.mtls.rejected", unit="{connection}",
description="API mTLS connections closed by the identity check, by reason")
def identity_check(cert: dict) -> tuple:
"""(allowed, reason, identity, serial) for a validated peer certificate dict (ssl.getpeercert())."""
sans = [value for kind, value in cert.get("subjectAltName", ()) if kind == "DNS"]
serial = str(cert.get("serialNumber", "")).upper()
identity = sans[0] if sans else "-"
if serial in REVOKED.current():
return False, "revoked", identity, serial
if not config.ALLOWED_CLIENT_IDENTITIES.intersection(sans):
return False, "identity_not_allowed", identity, serial
return True, "ok", identity, serial
class MutualTLSProtocol(HttpToolsProtocol):
def connection_made(self, transport) -> None:
super().connection_made(transport)
ssl_object = transport.get_extra_info("ssl_object")
cert = ssl_object.getpeercert() if ssl_object is not None else None
allowed, reason, identity, serial = identity_check(cert or {}) if cert else (False, "no_certificate", "-", "-")
if not allowed:
rejected.add(1, {"reason": reason})
log.warning("mtls_client_rejected reason=%s identity=%s serial=%s", reason, identity, serial)
transport.abort()
def ssl_context() -> ssl.SSLContext:
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=config.TLS_CA_FILE)
ctx.load_cert_chain(config.TLS_SERVER_CERT_FILE, config.TLS_SERVER_KEY_FILE)
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
return ctx
class Config(uvicorn.Config):
def load(self) -> None:
super().load()
self.ssl = ssl_context() # replace uvicorn's context: CA + CERT_REQUIRED + TLS >= 1.2
def main() -> None:
cfg = Config("app.main:app", host=os.environ.get("APP_HOST", "0.0.0.0"), port=int(os.environ.get("APP_PORT", "8080")),
http=MutualTLSProtocol, access_log=False, timeout_graceful_shutdown=int(config.GRACEFUL_TIMEOUT),
ssl_certfile=config.TLS_SERVER_CERT_FILE, ssl_keyfile=config.TLS_SERVER_KEY_FILE,
ssl_ca_certs=config.TLS_CA_FILE, ssl_cert_reqs=ssl.CERT_REQUIRED)
uvicorn.Server(cfg).run()
if __name__ == "__main__":
main()
+211
View File
@@ -0,0 +1,211 @@
import hashlib
import json
import logging
import secrets
import time
from datetime import datetime, timedelta, timezone
from typing import Optional
import valkey.asyncio as valkey
from app import config, repo
from app.cursor import fmt
from app.errors import unavailable
log = logging.getLogger("messenger.sessions")
REVOKED = "revoked"
class SessionStore:
def __init__(self, db) -> None:
self.db = db
self.client = None
self.stats = {"hits": 0, "misses": 0, "errors": 0}
self._local = {}
async def open(self) -> None:
if not config.CACHE_URL:
log.info("session cache disabled: CACHE_URL is empty")
return
# lab7: rediss:// = TLS to Valkey, server verified against the lab CA (hostname "valkey"); ACL user `api`.
self.client = valkey.Valkey.from_url(
config.CACHE_URL,
socket_timeout=config.CACHE_TIMEOUT,
socket_connect_timeout=config.CACHE_TIMEOUT,
decode_responses=True,
ssl_ca_certs=config.TLS_CA_FILE,
ssl_cert_reqs="required",
ssl_check_hostname=True,
)
async def close(self) -> None:
if self.client is not None:
await self.client.aclose()
self.client = None
@staticmethod
def digest(token: str) -> bytes:
return hashlib.sha256(token.encode("utf-8")).digest()
@staticmethod
def key(digest: bytes) -> str:
return "session:" + digest.hex()
async def create(self, user: dict) -> tuple:
token = secrets.token_urlsafe(32)
digest = self.digest(token)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
async with self.db.connection() as conn:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
" VALUES (%s, %s, clock_timestamp(), %s)",
(digest, user["id"], expires_at),
)
await self._cache_put(digest, user, expires_at)
return token, fmt(expires_at)
async def resolve(self, token: str) -> Optional[dict]:
digest = self.digest(token)
cached = await self._cache_get(digest)
if cached == REVOKED:
self.stats["hits"] += 1
return None
if cached is not None:
self.stats["hits"] += 1
return cached
self.stats["misses"] += 1
async with self.db.connection() as conn:
cur = await conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
" FROM sessions s JOIN users u ON u.id = s.user_id"
" WHERE s.token_hash = %s AND s.expires_at > now()",
(digest,),
)
row = await cur.fetchone()
if row is None:
return None
user = repo.public_user(row)
await self._cache_put(digest, user, row["expires_at"])
return user
async def revoke(self, token: str) -> None:
digest = self.digest(token)
await self._cache_revoke(digest)
try:
async with self.db.connection() as conn:
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
except Exception:
await self._cache_release(digest)
raise
async def sweep(self) -> int:
async with self.db.connection() as conn:
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
return cur.rowcount
def _cache_ttl(self, expires_at: datetime) -> int:
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
async def _cache_get(self, digest: bytes) -> Optional[dict]:
if self.client is None:
return None
try:
raw = await self.client.get(self.key(digest))
except Exception:
self._cache_failed("get")
return None
if not raw:
return None
if raw == REVOKED:
return REVOKED
try:
return json.loads(raw)
except ValueError:
return None
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
if self.client is None:
return
ttl = self._cache_ttl(expires_at)
if ttl <= 0:
return
payload = json.dumps(user)
try:
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
except Exception:
self._cache_failed("set")
async def _cache_release(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.delete(self.key(digest))
except Exception:
self._cache_failed("release")
async def _cache_revoke(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
except Exception:
self._cache_failed("revoke")
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
def _cache_failed(self, operation: str) -> None:
self.stats["errors"] += 1
if self.stats["errors"] % 50 == 1:
log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"])
# ------------------------------------------------------------------ lab7: shared login rate limit
async def hit(self, key: str, window: int) -> tuple:
"""INCR a fixed-window counter shared by all replicas. Returns (count, retry_after_seconds).
Valkey unavailable -> per-process fallback window (limits stay on, but are not shared)."""
if self.client is not None:
try:
count = await self.client.incr(key)
if count == 1:
await self.client.expire(key, window)
ttl = window
else:
ttl = await self.client.ttl(key)
if ttl < 0: # lost EXPIRE (crash between INCR and EXPIRE): repair
await self.client.expire(key, window)
ttl = window
return count, max(1, ttl)
except Exception:
self._cache_failed("ratelimit")
return self._local_hit(key, window)
async def peek(self, key: str) -> tuple:
if self.client is not None:
try:
raw = await self.client.get(key)
ttl = await self.client.ttl(key) if raw else 0
return int(raw or 0), max(1, ttl)
except Exception:
self._cache_failed("ratelimit")
entry = self._local.get(key)
if entry and entry[1] > time.monotonic():
return entry[0], max(1, int(entry[1] - time.monotonic()))
return 0, 1
async def reset(self, key: str) -> None:
self._local.pop(key, None)
if self.client is not None:
try:
await self.client.delete(key)
except Exception:
self._cache_failed("ratelimit")
def _local_hit(self, key: str, window: int) -> tuple:
now = time.monotonic()
count, until = self._local.get(key, (0, now + window))
if until <= now:
count, until = 0, now + window
self._local[key] = (count + 1, until)
if len(self._local) > 10000: # bounded memory under a username-spraying attack
self._local = {k: v for k, v in self._local.items() if v[1] > now}
return count + 1, max(1, int(until - now))
+393
View File
@@ -0,0 +1,393 @@
"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
"""
import json
import logging
import os
import re
import resource
import sys
import time
from contextlib import contextmanager
from typing import Optional
from opentelemetry import context as otel_context
from opentelemetry import metrics, propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import config
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
QUEUE = config.BROKER_QUEUE
JOB_TYPE = "attachment.process.v1"
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
tracer = trace.get_tracer("messenger")
meter = metrics.get_meter("messenger")
_state = {"service": "messenger", "providers": []}
# ---------------------------------------------------------------------------------------------- logging
class EventFilter(logging.Filter):
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
def filter(self, record: logging.LogRecord) -> bool:
if not getattr(record, "event", None):
match = self.EVENT_RE.match(str(record.msg))
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
return True
class JsonFormatter(logging.Formatter):
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
def format(self, record: logging.LogRecord) -> str:
out = {
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
"level": record.levelname,
"service": _state["service"],
"event": getattr(record, "event", ""),
"logger": record.name,
"msg": record.getMessage(),
}
ctx = trace.get_current_span().get_span_context()
if ctx.is_valid:
out["trace_id"] = format(ctx.trace_id, "032x")
out["span_id"] = format(ctx.span_id, "016x")
if record.exc_info:
out["exc"] = self.formatException(record.exc_info)
return json.dumps(out, ensure_ascii=False)
def _setup_logging(logger_provider=None) -> None:
root = logging.getLogger()
for handler in list(root.handlers):
root.removeHandler(handler)
root.setLevel(logging.INFO)
stdout = logging.StreamHandler(sys.stdout)
stdout.setFormatter(JsonFormatter())
stdout.addFilter(EventFilter())
root.addHandler(stdout)
# The exporter's own warnings (backend down) must not be fed back into the exporter.
sdk_log = logging.getLogger("opentelemetry")
sdk_log.propagate = False
sdk_log.addHandler(stdout)
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
logging.getLogger(noisy).setLevel(logging.WARNING)
if logger_provider is not None:
from opentelemetry.sdk._logs import LoggingHandler
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otlp.addFilter(EventFilter())
root.addHandler(otlp)
# ---------------------------------------------------------------------------------------------- sampling
def _build_sampler():
"""ParentBased(root=KindFilter(TraceIdRatio)).
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
Children always follow the parent's decision, so an upload and its worker job are sampled together.
"""
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
TraceIdRatioBased)
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
decisions = meter.create_counter(
"messenger.telemetry.sampler.decisions", unit="{span}",
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
class KindFilter(Sampler):
def __init__(self, delegate):
self.delegate = delegate
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
return SamplingResult(Decision.DROP)
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
trace_state)
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
return result
def get_description(self):
return "KindFilter{%s}" % self.delegate.get_description()
class CountingParentBased(ParentBased):
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
parent = trace.get_current_span(parent_context).get_span_context()
if parent.is_valid:
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
"rule": "parent"})
return result
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
class _CountingExporter:
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
def __init__(self, delegate):
from opentelemetry.sdk.trace.export import SpanExportResult
self.delegate = delegate
self.ok = SpanExportResult.SUCCESS
self.counter = meter.create_counter(
"messenger.telemetry.spans.exported", unit="{span}",
description="Spans handed to the OTLP exporter by outcome (success/failure)")
def export(self, spans):
result = self.delegate.export(spans)
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
return result
def shutdown(self):
return self.delegate.shutdown()
def force_flush(self, timeout_millis: int = 30000):
return self.delegate.force_flush(timeout_millis)
# ---------------------------------------------------------------------------------------------- setup
def setup(service_name: str, fastapi_app=None) -> None:
_state["service"] = service_name
if not ENABLED:
_setup_logging(None)
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
return
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
res = Resource.create({
"service.name": service_name,
"service.namespace": "messenger",
"service.version": SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
})
views = [
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
View(instrument_name="http.server.request.duration",
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.attachment.time_to_ready",
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
# No catch-all View("*"): an instrument matched by two views would be exported twice.
# Instruments without a matching view get the SDK default aggregation.
]
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
views=views)
metrics.set_meter_provider(meter_provider)
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
trace.set_tracer_provider(tracer_provider)
logger_provider = LoggerProvider(resource=res)
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
_setup_logging(logger_provider)
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
# placeholders), so values, message bodies and hashes never reach db.query.text.
BotocoreInstrumentor().instrument()
if fastapi_app is not None:
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
exclude_spans=["receive", "send"])
_process_metrics()
logging.getLogger("messenger.telemetry").info(
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
def shutdown() -> None:
for provider in _state["providers"]:
try:
provider.shutdown()
except Exception:
pass
# ---------------------------------------------------------------------------------------------- context carrier
def inject() -> dict:
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
carrier: dict = {}
if trace.get_current_span().get_span_context().is_valid:
propagate.inject(carrier)
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
def extract(carrier: Optional[dict]):
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
return otel_context.Context()
clean = {"traceparent": str(carrier["traceparent"])}
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
clean["tracestate"] = carrier["tracestate"]
return propagate.extract(clean, context=otel_context.Context())
def set_attributes(**attrs) -> None:
span = trace.get_current_span()
for key, value in attrs.items():
if value is not None:
span.set_attribute(key.replace("__", "."), value)
@contextmanager
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
record_exception=True, set_status_on_exception=True) as current:
yield current
def record_exception(exc: BaseException) -> None:
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
current = trace.get_current_span()
current.record_exception(exc)
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
def mark_error(current, description: str) -> None:
current.set_status(Status(StatusCode.ERROR, description))
# ---------------------------------------------------------------------------------------------- instruments
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
def _rss_bytes() -> int:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE
except OSError:
return 0
def _process_metrics() -> None:
from opentelemetry.metrics import Observation
def cpu(_options):
own = resource.getrusage(resource.RUSAGE_SELF)
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
description="Resident set size of the process")
def register_pool(db) -> None:
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def count(_options):
stats = db.pool.get_stats()
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
yield Observation(idle, {"db.client.connection.state": "idle"})
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
meter.create_observable_up_down_counter(
"db.client.connection.pending_requests",
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
meter.create_observable_up_down_counter(
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
def register_queue(state: dict) -> None:
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
"""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def depth(_options):
queue = state.get("queue")
if queue is None:
return []
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
def oldest(_options):
queue = state.get("queue")
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
description="Jobs accepted but not finished, by state")
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
description="Age of the oldest queued/processing job since durable acceptance")
def register_worker(worker) -> None:
if not ENABLED:
return
from opentelemetry.metrics import Observation
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
unit="{job}", description="Jobs being processed right now")
meter.create_observable_gauge("messenger.worker.capacity",
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
description="Concurrent job slots of this worker")
jobs_processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
job_duration = meter.create_histogram(
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
time_to_ready = meter.create_histogram(
"messenger.attachment.time_to_ready", unit="s",
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")
+373
View File
@@ -0,0 +1,373 @@
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
Per delivery:
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
3. download staging object, verify size+sha256 against DB metadata
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
7. delete staging (best effort, reconciler finishes it), then ack
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
"""
import asyncio
import hashlib
import json
import logging
import os
import resource
import signal
import sys
import tempfile
import time
import uuid
from datetime import datetime, timezone
from opentelemetry.trace import SpanKind
from app import config, faults, jobs, openbao, telemetry
from app.broker import connect, get_queue
from app.db import Database
from app.s3 import S3Storage
log = logging.getLogger("messenger.worker")
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
class Permanent(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class Transient(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class LeaseLost(Exception):
pass
def child_limits() -> None:
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
cpu = config.JOB_TIMEOUT_SECONDS
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
os.nice(5)
def sha256_file(path: str):
digest, size = hashlib.sha256(), 0
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
size += len(chunk)
return digest.hexdigest(), size
class Worker:
def __init__(self) -> None:
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
self.db = Database()
self.s3 = S3Storage()
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
self.stopping = asyncio.Event()
self.inflight = 0
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
# ------------------------------------------------------------------ lifecycle
async def run(self) -> None:
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
await self.db.open()
telemetry.register_pool(self.db)
telemetry.register_worker(self)
await asyncio.to_thread(self.s3.open)
bao = openbao.client()
if bao is not None:
# Rotation of the worker's DB password / S3 key in OpenBao is picked up without a restart.
bao.on_change(lambda data: self.db.rotate(data.get("database_url", "")))
bao.on_change(self.s3.rotate)
bao.start_refresher(config.BAO_REFRESH_SECONDS)
connection = await connect()
channel = await connection.channel()
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
queue = await get_queue(channel)
tag = await queue.consume(self.on_message, no_ack=False)
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
await self.stopping.wait()
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
try:
await queue.cancel(tag)
except Exception:
pass
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
while self.inflight and time.monotonic() < deadline:
await asyncio.sleep(0.2)
for task in background:
task.cancel()
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
await connection.close()
await self.db.close()
log.info("worker_stopped stats=%s", self.stats)
async def alive(self) -> None:
while True:
try:
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
handle.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(2)
async def sweeper(self) -> None:
while True:
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
try:
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
if swept:
log.warning("lease_expired recovered=%s", swept)
except Exception:
log.warning("lease_sweep_skipped: database unavailable")
# ------------------------------------------------------------------ delivery
async def on_message(self, message) -> None:
async with self.slots:
self.inflight += 1
started = time.monotonic()
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
try:
carrier = json.loads(message.body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
carrier = None
if not isinstance(carrier, dict) or "traceparent" not in carrier:
carrier = dict(message.headers or {})
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
"messaging.destination.name": config.BROKER_QUEUE,
"messaging.message.id": str(message.message_id or ""),
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
outcome = "requeued"
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
context=telemetry.extract(carrier), attributes=attrs) as current:
try:
outcome = await self.handle(message)
except Exception:
log.exception("unexpected_error, message returned to queue")
await asyncio.sleep(1)
await self.settle(message.nack(requeue=True))
finally:
self.inflight -= 1
current.set_attribute("messenger.job.outcome", outcome)
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
telemetry.mark_error(current, outcome)
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
telemetry.jobs_processed.add(1, labels)
telemetry.job_duration.record(time.monotonic() - started, labels)
@staticmethod
async def settle(awaitable) -> None:
try:
await awaitable
except Exception:
log.warning("could not settle delivery (connection lost); broker will redeliver")
async def handle(self, message) -> str:
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
msg = jobs.parse_message(message.body)
if msg is None:
self.stats["dlq"] += 1
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
await self.settle(message.reject(requeue=False))
return "dlq"
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
try:
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
except Exception:
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
if decision == "done" and job["state"] == "succeeded":
await self.cleanup_staging(job["attachment_id"])
if decision != "claimed":
self.stats["noop"] += 1
telemetry.set_attributes(messenger__job__decision=decision)
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
message.redelivered)
await self.settle(message.ack())
return "noop"
started = time.monotonic()
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
job["max_attempts"])
lease_lost = asyncio.Event()
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
outcome = None
work = asyncio.create_task(self.process(job, message))
lost = asyncio.create_task(lease_lost.wait())
try:
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
return_when=asyncio.FIRST_COMPLETED)
if work not in done:
work.cancel()
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
outcome = "succeeded" if work.result() else "lost"
except LeaseLost:
outcome = "lost"
except Permanent as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
except Transient as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
except Exception as exc: # S3/DB/network errors
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
finally:
keeper.cancel()
lost.cancel()
if outcome is None:
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
self.stats[outcome] = self.stats.get(outcome, 0) + 1
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
time.monotonic() - started)
await self.settle(message.ack())
return outcome
async def cleanup_staging(self, attachment_id) -> None:
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
try:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, attachment_id)
if src and src["status"] == "ready" and src["staging_key"]:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
while True:
await asyncio.sleep(interval)
try:
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
except Exception:
continue # DB blip: keep trying until the lease really expires
if not alive:
log.warning("lease_lost job=%s: another worker owns it now", job_id)
lease_lost.set()
return
async def record_failure(self, job, code: str, detail: str, transient: bool):
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
transient)
try:
return await self.db.in_transaction(
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
except Exception:
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
return None
# ------------------------------------------------------------------ the actual work
async def process(self, job, message) -> bool:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, job["attachment_id"])
if src is None or not src["staging_key"]:
raise Permanent("staging_missing", "attachment has no staging object")
filename = src["filename"]
first_delivery = job["attempt"] == 1 and not message.redelivered
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
if faults.active(filename, "permanent"):
raise Permanent("invalid_image", "injected permanent failure")
if job["attempt"] <= faults.transient_until(filename):
raise Transient("dependency_unavailable", "injected transient failure")
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
source = os.path.join(tmp, "source")
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
raise Permanent("staging_missing", "staging object not found")
digest, size = await asyncio.to_thread(sha256_file, source)
if digest != src["sha256"] or size != src["size_bytes"]:
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
out_dir = os.path.join(tmp, "out")
os.mkdir(out_dir)
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
produced = await self.run_child(source, out_dir)
variants = []
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
for item in produced:
digest, size = await asyncio.to_thread(sha256_file, item["file"])
key = prefix + os.path.basename(item["file"])
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
raise Permanent("processing_failed", "not all variants were produced")
if first_delivery and faults.active(filename, "worker_crash_after_put"):
faults.crash("after S3 put, before result COMMIT")
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
if not committed:
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
return False
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
{"job.type": telemetry.JOB_TYPE})
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
faults.crash("after result COMMIT, before ack")
try:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
return True
async def run_child(self, source: str, out_dir: str) -> list:
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
proc = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
)
try:
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
except asyncio.CancelledError:
proc.kill()
raise
code = proc.returncode
if code == 0:
return json.loads(stdout)
if code in PERMANENT_EXIT:
raise Permanent(PERMANENT_EXIT[code])
if code == -signal.SIGXCPU:
raise Permanent("image_too_complex", "CPU limit exceeded")
if code == -signal.SIGKILL:
raise Transient("worker_resource_exhausted", "child killed (OOM)")
raise Permanent("processing_failed", "child exit %s" % code)
def main() -> None:
telemetry.setup("messenger-worker")
try:
asyncio.run(Worker().run())
finally:
telemetry.shutdown()
if __name__ == "__main__":
main()
+604
View File
@@ -0,0 +1,604 @@
name: mtusi-lab7
# Trust layout: contracts/trust-matrix.md. Every TLS key reaches only its own container (Docker secrets from
# secrets/pki, created by scripts/pki.sh). API and worker receive NO passwords: they authenticate to OpenBao with
# their client certificate and read their scoped KV secret (scripts/bao.sh bootstrap).
x-hardened: &hardened
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
restart: unless-stopped
x-otel-env: &otel-env
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca.crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel.crt
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel.key
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_SEMCONV_STABILITY_OPT_IN: http
OTEL_METRIC_EXPORT_INTERVAL: "10000"
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
OTEL_BSP_SCHEDULE_DELAY: "2000"
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
SERVICE_VERSION: "1.7.0"
x-app-env: &app-env
TLS_CA_FILE: /run/secrets/ca.crt
TLS_CLIENT_CERT_FILE: /run/secrets/client.crt
TLS_CLIENT_KEY_FILE: /run/secrets/client.key
BAO_ADDR: https://openbao:8200
S3_ENDPOINT: https://minio:9000
BROKER_URL: amqps://rabbitmq:5671/messenger?auth=external&heartbeat=30
DATABASE_SSLMODE: verify-full
x-api: &api
<<: *hardened
build:
context: .
image: mtusi-messenger:lab7
env_file: [.env]
tmpfs: ["/tmp:size=128m,mode=1777"]
depends_on:
postgres: {condition: service_healthy}
rabbitmq: {condition: service_healthy}
valkey: {condition: service_healthy}
openbao: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
migrate: {condition: service_completed_successfully}
deploy:
resources:
limits: {cpus: "0.50", memory: 512M, pids: 256}
# Business port 8080 is mTLS-only; the healthcheck uses the loopback-only health listener 127.0.0.1:8081.
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
services:
nginx:
<<: *hardened
image: nginx:1.30.6-alpine@sha256:583b7f2fce1cdb8da5ad5c39ebd3c33b98b87d493190e548bdee1fa8c619c311
user: "101:101"
command: ["/bin/sh", "/opt/nginx-health.sh"]
environment:
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
HEALTH_INTERVAL: "1"
HEALTH_FALL: "2"
HEALTH_RISE: "2"
ports:
- "127.0.0.1:8443:8443" # the only business ingress: TLS
tmpfs: ["/tmp:uid=101,gid=101,mode=0700", "/var/cache/nginx:uid=101,gid=101,mode=0700",
"/var/run:uid=101,gid=101,mode=0755", "/etc/nginx/upstream:uid=101,gid=101,mode=0755"]
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
secrets:
- {source: ca_crt, target: ca.crt}
- {source: lb_server_crt, target: server.crt}
- {source: lb_server_key, target: server.key}
- {source: lb_client_crt, target: client.crt}
- {source: lb_client_key, target: client.key}
depends_on:
api-1: {condition: service_healthy}
api-2: {condition: service_healthy}
deploy:
resources:
limits: {cpus: "0.25", memory: 64M, pids: 256}
healthcheck:
test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://localhost:8443/health/live >/dev/null"]
interval: 5s
timeout: 3s
retries: 3
start_period: 5s
api-1:
<<: *api
environment:
<<: [*otel-env, *app-env]
INSTANCE_ID: api-1
LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
BAO_AUTH_ROLE: api
BAO_KV_PATH: api
TLS_SERVER_CERT_FILE: /run/secrets/server.crt
TLS_SERVER_KEY_FILE: /run/secrets/server.key
ALLOWED_CLIENT_IDENTITIES: lb.client.messenger
REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
secrets: &api-secrets
- {source: ca_crt, target: ca.crt}
- {source: api_1_crt, target: server.crt}
- {source: api_1_key, target: server.key}
- {source: api_client_crt, target: client.crt}
- {source: api_client_key, target: client.key}
- {source: api_otel_crt, target: otel.crt}
- {source: api_otel_key, target: otel.key}
- {source: revoked_serials, target: revoked-serials.txt}
api-2:
<<: *api
environment:
<<: [*otel-env, *app-env]
INSTANCE_ID: api-2
LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
BAO_AUTH_ROLE: api
BAO_KV_PATH: api
TLS_SERVER_CERT_FILE: /run/secrets/server.crt
TLS_SERVER_KEY_FILE: /run/secrets/server.key
ALLOWED_CLIENT_IDENTITIES: lb.client.messenger
REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
secrets:
- {source: ca_crt, target: ca.crt}
- {source: api_2_crt, target: server.crt}
- {source: api_2_key, target: server.key}
- {source: api_client_crt, target: client.crt}
- {source: api_client_key, target: client.key}
- {source: api_otel_crt, target: otel.crt}
- {source: api_otel_key, target: otel.key}
- {source: revoked_serials, target: revoked-serials.txt}
worker:
<<: *hardened
build:
context: .
image: mtusi-messenger:lab7
command: ["python", "-m", "app.worker"]
env_file: [.env]
environment:
<<: [*otel-env, *app-env]
INSTANCE_ID: ""
BAO_AUTH_ROLE: worker
BAO_KV_PATH: worker
DB_POOL_MAX: "6"
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
tmpfs: ["/tmp:size=512m,mode=1777"]
secrets:
- {source: ca_crt, target: ca.crt}
- {source: worker_client_crt, target: client.crt}
- {source: worker_client_key, target: client.key}
- {source: worker_otel_crt, target: otel.crt}
- {source: worker_otel_key, target: otel.key}
depends_on:
postgres: {condition: service_healthy}
rabbitmq: {condition: service_healthy}
openbao: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
migrate: {condition: service_completed_successfully}
stop_grace_period: 30s
deploy:
replicas: ${WORKER_REPLICAS:-2}
resources:
limits: {cpus: "${WORKER_CPUS:-2.00}", memory: 1536M, pids: 256}
healthcheck:
test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
migrate:
<<: *hardened
build:
context: .
image: mtusi-messenger:lab7
command: ["python", "scripts/migrate.py"]
restart: "no"
environment:
DATABASE_URL_FILE: /run/secrets/migrate_database_url
DATABASE_SSLMODE: verify-full
TLS_CA_FILE: /run/secrets/ca.crt
DB_API_PASSWORD_FILE: /run/secrets/db_api_password
DB_WORKER_PASSWORD_FILE: /run/secrets/db_worker_password
secrets:
- {source: ca_crt, target: ca.crt}
- migrate_database_url
- db_api_password
- db_worker_password
depends_on:
postgres: {condition: service_healthy}
deploy:
resources:
limits: {cpus: "0.25", memory: 128M, pids: 256}
openbao:
<<: *hardened
image: openbao/openbao:2.6.5@sha256:09f1eaaee351c329d75a5a89654d6e1ad4792998650c3cbf0c3513a76d9d9b6e
user: "100:1000"
entrypoint: ["bao", "server", "-config=/openbao/config/config.hcl"]
tmpfs: ["/tmp"]
volumes:
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
- baodata:/openbao/file
secrets:
- {source: ca_crt, target: ca.crt}
- {source: openbao_crt, target: openbao.crt}
- {source: openbao_key, target: openbao.key}
deploy:
resources:
limits: {cpus: "0.50", memory: 256M, pids: 256}
# Healthy = server answers (sealed or not). Services retry their login while it is sealed.
healthcheck:
test: ["CMD-SHELL", "BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/run/secrets/ca.crt bao status >/dev/null 2>&1; [ $$? -ne 1 ]"]
interval: 5s
timeout: 3s
retries: 20
postgres:
<<: *hardened
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
user: "70:70"
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_INITDB_ARGS: --data-checksums --auth-host=scram-sha-256
# PostgreSQL refuses a key file readable by others: copy it to the tmpfs with mode 0600 first.
entrypoint: ["sh", "-c", "install -m 600 /run/secrets/server.key /tmp/server.key && exec docker-entrypoint.sh \"$$@\"", "--"]
command:
- postgres
- -c
- ssl=on
- -c
- ssl_cert_file=/run/secrets/server.crt
- -c
- ssl_key_file=/tmp/server.key
- -c
- ssl_min_protocol_version=TLSv1.2
- -c
- hba_file=/etc/postgresql/pg_hba.conf
- -c
- password_encryption=scram-sha-256
- -c
- synchronous_commit=on
- -c
- shared_buffers=128MB
- -c
- max_connections=60
tmpfs: ["/tmp", "/var/run/postgresql"]
secrets:
- postgres_password
- {source: postgres_crt, target: server.crt}
- {source: postgres_key, target: server.key}
volumes:
- pgdata:/var/lib/postgresql/data
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h /var/run/postgresql"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
valkey:
<<: *hardened
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
user: "999:1000"
entrypoint: ["sh", "/opt/valkey-start.sh"]
tmpfs: ["/tmp", "/data"]
volumes:
- ./infra/valkey/valkey-start.sh:/opt/valkey-start.sh:ro
secrets:
- {source: ca_crt, target: ca.crt}
- {source: valkey_crt, target: server.crt}
- {source: valkey_key, target: server.key}
- valkey_api_password
deploy:
resources:
limits: {cpus: "0.25", memory: 128M, pids: 256}
healthcheck:
test: ["CMD", "valkey-cli", "--tls", "--cacert", "/run/secrets/ca.crt", "--user", "health", "--pass", "health", "-h", "valkey", "ping"]
interval: 5s
timeout: 3s
retries: 10
rabbitmq:
<<: *hardened
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
user: "100:101"
hostname: rabbitmq
environment:
RABBITMQ_ENABLED_PLUGINS_FILE: /etc/rabbitmq/enabled_plugins
# Erlang distribution + epmd loopback-only (no cluster; CLI runs inside the container). The node name must
# then resolve to loopback, hence rabbit@localhost (its data dir is /var/lib/rabbitmq/mnesia/rabbit@localhost).
RABBITMQ_NODENAME: rabbit@localhost
ERL_EPMD_ADDRESS: 127.0.0.1
tmpfs: ["/tmp", "/etc/rabbitmq/conf.d"]
volumes:
- rabbitmqdata:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
- ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro
secrets:
- {source: ca_crt, target: ca.crt}
- {source: rabbitmq_crt, target: server.crt}
- {source: rabbitmq_key, target: server.key}
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
interval: 5s
timeout: 10s
retries: 30
start_period: 20s
minio:
<<: *hardened
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
command: ["server", "/data", "--certs-dir", "/tmp/certs"]
entrypoint: ["sh", "-c", "mkdir -p /tmp/certs/CAs && cp /run/secrets/server.crt /tmp/certs/public.crt && cp /run/secrets/server.key /tmp/certs/private.key && cp /run/secrets/ca.crt /tmp/certs/CAs/ca.crt && exec minio \"$$@\"", "--"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
MINIO_BROWSER: "off"
tmpfs: ["/tmp"]
secrets:
- minio_root_user
- minio_root_password
- {source: ca_crt, target: ca.crt}
- {source: minio_crt, target: server.crt}
- {source: minio_key, target: server.key}
volumes:
- miniodata:/data
deploy:
resources:
limits: {cpus: "0.50", memory: 512M, pids: 256}
healthcheck:
test: ["CMD-SHELL", "mkdir -p /tmp/hc/certs/CAs && cp /run/secrets/ca.crt /tmp/hc/certs/CAs/ca.crt && MC_HOST_hc=https://minio:9000 mc --config-dir /tmp/hc ready hc >/dev/null"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
minio-init:
<<: *hardened
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
entrypoint: ["/bin/sh", "/opt/init.sh"]
restart: "no"
environment:
S3_BUCKET: messenger-attachments
MC_CONFIG_DIR: /tmp/mc
tmpfs: ["/tmp"]
secrets:
- minio_root_user
- minio_root_password
- {source: ca_crt, target: ca.crt}
- s3_api_access_key
- s3_api_secret_key
- s3_worker_access_key
- s3_worker_secret_key
volumes:
- ./infra/minio/init.sh:/opt/init.sh:ro
- ./infra/minio/policy-api.json:/etc/minio-policy/policy-api.json:ro
- ./infra/minio/policy-worker.json:/etc/minio-policy/policy-worker.json:ro
depends_on:
minio: {condition: service_healthy}
# ------------------------------------------------------------------ observability (lab6) behind mTLS
otel-collector:
<<: *hardened
image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917
command:
- --config=/etc/otelcol/collector.yaml
- --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
- ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro
- ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro
secrets:
- {source: ca_crt, target: ca.crt}
- {source: ingest_ca_crt, target: ingest-ca.crt}
- {source: otel_collector_crt, target: server.crt}
- {source: otel_collector_key, target: server.key}
- {source: collector_out_crt, target: client.crt}
- {source: collector_out_key, target: client.key}
deploy:
resources:
limits: {cpus: "1.00", memory: 256M, pids: 256}
prometheus:
<<: *hardened
image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62
command:
- --config.file=/etc/prometheus/prometheus.yml
- --web.config.file=/etc/prometheus/web.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
- ./infra/prometheus/health-client.yml:/etc/prometheus/health-client.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- promdata:/prometheus
secrets:
- {source: ca_crt, target: ca.crt}
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
- {source: prometheus_crt, target: server.crt}
- {source: prometheus_key, target: server.key}
- {source: operator_crt, target: health.crt}
- {source: operator_key, target: health.key}
ports:
- "127.0.0.1:9090:9090" # mTLS: operator certificate only
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
healthcheck:
test: ["CMD", "promtool", "check", "healthy", "--url=https://prometheus:9090", "--http.config.file=/etc/prometheus/health-client.yml"]
interval: 5s
timeout: 3s
retries: 20
loki:
<<: *hardened
image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
secrets:
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
- {source: loki_crt, target: server.crt}
- {source: loki_key, target: server.key}
ports:
- "127.0.0.1:3100:3100" # mTLS: operator certificate only
deploy:
resources:
limits: {cpus: "1.00", memory: 512M, pids: 256}
healthcheck: # TCP only: every HTTP request needs a telemetry client certificate
test: ["CMD", "/busybox/nc", "-z", "127.0.0.1", "3100"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
tempo:
<<: *hardened
image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69
command: ["-config.file=/etc/tempo/tempo.yaml"]
tmpfs: ["/tmp"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
secrets:
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
- {source: tempo_crt, target: server.crt}
- {source: tempo_key, target: server.key}
ports:
- "127.0.0.1:3200:3200" # mTLS: operator certificate only
deploy:
resources:
limits: {cpus: "1.00", memory: 1024M, pids: 256}
healthcheck:
test: ["CMD", "nc", "-z", "127.0.0.1", "3200"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
grafana:
<<: *hardened
image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7
environment:
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/secrets/server.crt
GF_SERVER_CERT_KEY: /run/secrets/server.key
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_SECURITY_COOKIE_SECURE: "true"
GF_USERS_ALLOW_SIGN_UP: "false"
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
tmpfs: ["/tmp"]
secrets:
- grafana_admin_password
- {source: ca_crt, target: ca.crt}
- {source: grafana_crt, target: server.crt}
- {source: grafana_key, target: server.key}
- {source: grafana_out_crt, target: client.crt}
- {source: grafana_out_key, target: client.key}
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000" # HTTPS + admin login
depends_on:
prometheus: {condition: service_healthy}
loki: {condition: service_started}
tempo: {condition: service_started}
deploy:
resources:
limits: {cpus: "0.50", memory: 256M, pids: 256}
healthcheck:
test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://grafana:3000/api/health >/dev/null"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
volumes:
pgdata:
miniodata:
rabbitmqdata:
baodata:
promdata:
lokidata:
tempodata:
grafanadata:
secrets:
ca_crt: {file: ./secrets/pki/ca.crt}
ingest_ca_crt: {file: ./secrets/pki/ingest.crt}
telemetry_ca_crt: {file: ./secrets/pki/telemetry.crt}
revoked_serials: {file: ./secrets/pki/revoked-serials.txt}
lb_server_crt: {file: ./secrets/pki/lb-server.crt}
lb_server_key: {file: ./secrets/pki/lb-server.key}
lb_client_crt: {file: ./secrets/pki/lb-client.crt}
lb_client_key: {file: ./secrets/pki/lb-client.key}
api_1_crt: {file: ./secrets/pki/api-1.crt}
api_1_key: {file: ./secrets/pki/api-1.key}
api_2_crt: {file: ./secrets/pki/api-2.crt}
api_2_key: {file: ./secrets/pki/api-2.key}
api_client_crt: {file: ./secrets/pki/api.crt}
api_client_key: {file: ./secrets/pki/api.key}
worker_client_crt: {file: ./secrets/pki/worker.crt}
worker_client_key: {file: ./secrets/pki/worker.key}
api_otel_crt: {file: ./secrets/pki/api-otel.crt}
api_otel_key: {file: ./secrets/pki/api-otel.key}
worker_otel_crt: {file: ./secrets/pki/worker-otel.crt}
worker_otel_key: {file: ./secrets/pki/worker-otel.key}
postgres_crt: {file: ./secrets/pki/postgres.crt}
postgres_key: {file: ./secrets/pki/postgres.key}
minio_crt: {file: ./secrets/pki/minio.crt}
minio_key: {file: ./secrets/pki/minio.key}
rabbitmq_crt: {file: ./secrets/pki/rabbitmq.crt}
rabbitmq_key: {file: ./secrets/pki/rabbitmq.key}
valkey_crt: {file: ./secrets/pki/valkey.crt}
valkey_key: {file: ./secrets/pki/valkey.key}
openbao_crt: {file: ./secrets/pki/openbao.crt}
openbao_key: {file: ./secrets/pki/openbao.key}
otel_collector_crt: {file: ./secrets/pki/otel-collector.crt}
otel_collector_key: {file: ./secrets/pki/otel-collector.key}
collector_out_crt: {file: ./secrets/pki/collector-out.crt}
collector_out_key: {file: ./secrets/pki/collector-out.key}
prometheus_crt: {file: ./secrets/pki/prometheus.crt}
prometheus_key: {file: ./secrets/pki/prometheus.key}
loki_crt: {file: ./secrets/pki/loki.crt}
loki_key: {file: ./secrets/pki/loki.key}
tempo_crt: {file: ./secrets/pki/tempo.crt}
tempo_key: {file: ./secrets/pki/tempo.key}
grafana_crt: {file: ./secrets/pki/grafana.crt}
grafana_key: {file: ./secrets/pki/grafana.key}
grafana_out_crt: {file: ./secrets/pki/grafana-out.crt}
grafana_out_key: {file: ./secrets/pki/grafana-out.key}
operator_crt: {file: ./secrets/pki/operator.crt}
operator_key: {file: ./secrets/pki/operator.key}
postgres_password: {file: ./secrets/bootstrap/postgres_password}
migrate_database_url: {file: ./secrets/bootstrap/migrate_database_url}
db_api_password: {file: ./secrets/bootstrap/db_api_password}
db_worker_password: {file: ./secrets/bootstrap/db_worker_password}
minio_root_user: {file: ./secrets/bootstrap/minio_root_user}
minio_root_password: {file: ./secrets/bootstrap/minio_root_password}
s3_api_access_key: {file: ./secrets/bootstrap/s3_api_access_key}
s3_api_secret_key: {file: ./secrets/bootstrap/s3_api_secret_key}
s3_worker_access_key: {file: ./secrets/bootstrap/s3_worker_access_key}
s3_worker_secret_key: {file: ./secrets/bootstrap/s3_worker_secret_key}
valkey_api_password: {file: ./secrets/bootstrap/valkey_api_password}
grafana_admin_password: {file: ./secrets/bootstrap/grafana_admin_password}
+73 -12
View File
@@ -4,18 +4,18 @@
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | `infra/nginx.conf` (listen 8443 ssl), `app/main.py` login; аудит I1–I4, `make test-security`, `test_auth_acl.py` |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | `app/serve.py` (CERT_REQUIRED + SAN allowlist + отозванные serial), `infra/nginx.conf` proxy_ssl_*; аудит A1–A14, `test_cert_rotation.py` |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | `infra/postgres/pg_hba.conf`, `migrations/0007_service_roles.sql`, `app/db.py` (verify-full); аудит B1–B14, `test_secret_rotation.py` |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | `infra/minio/policy-*.json`, `app/s3.py` (verify=CA); аудит C1–C14 |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | `infra/rabbitmq/rabbitmq.conf`, `definitions.json`, `app/broker.py`; аудит D1–D14 |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | `infra/otel/collector.yaml` (client_ca_file = ingest CA); аудит E1–E7 |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | `infra/prometheus/web.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, exporters в `collector.yaml`; аудит F1–F24 |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | `infra/grafana/provisioning/datasources/datasources.yaml` (tlsAuth + CA); аудит F (grafana-out), E (grafana-out отклонён), J1–J2 |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | `infra/openbao/config.hcl`, `policies/*.hcl`, `scripts/bao.sh`, `app/openbao.py`; аудит H1–H20, `test_bao_sealed.py` |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | `scripts/bao.sh`, `compose.yaml` (порты только 127.0.0.1); аудит J, H (admin-пути запрещены приложениям), `check_hardening.py` |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | `infra/valkey/valkey-start.sh` (TLS, ACL), `app/sessions.py`; аудит G1–G11 |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | N/A: репликации и кластеров нет (по одному узлу каждого хранилища). Erlang distribution RabbitMQ, внутренний gRPC Loki/Tempo — только loopback; аудит K1–K13 |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
@@ -35,3 +35,64 @@
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
---
# Заполнение (решение lab7)
Все отрицательные и положительные проверки ниже выполняет `student-tests/tls_audit.py` изнутри сети стенда
(разделы A–K, **135 проверок**). Сырой вывод: `evidence/tls_audit.txt`, `evidence/tls_audit.json`.
Запуск: `docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py`.
## PKI и identities
| CA | Кем подписан | Что подписывает | Кто доверяет |
| --- | --- | --- | --- |
| Messenger Lab Root CA (`ca.crt`, EC P-256, 825 дн.) | self-signed | все серверные сертификаты; клиентские identities `lb`, `api`, `worker`, `intruder` (тестовый) | все клиенты для проверки сервера; API (LB), RabbitMQ, OpenBao, PostgreSQL |
| Telemetry Ingest CA | Root | только `api.otel.messenger`, `worker.otel.messenger` | **только** Collector (client_ca_file) |
| Telemetry Clients CA | Root | только `collector.out`, `grafana.out`, `operator` | **только** Prometheus, Loki, Tempo |
| Foreign CA (тестовый) | self-signed | `foreign-client`, заявляет имена lb/api/worker | никто |
Leaf-сертификаты живут 90 дней (`scripts/pki.sh`), ключи EC P-256 лежат только в `secrets/pki/` (0700) и попадают в контейнер как Docker secret именно того сервиса. Отзыв: `revoked-serials.txt`, API перечитывает его на лету.
## Детальная матрица
| Связь | Порт / endpoint | Сервер (CA, SAN) | Клиентская identity | Проверка SAN / ACL | Срок жизни credentials | Положительный / отрицательный тест |
| --- | --- | --- | --- | --- | --- | --- |
| Клиент → nginx | `127.0.0.1:8443` HTTPS | lb-server: Root, SAN `localhost`, `nginx`, `127.0.0.1` | пользователь: Bearer (256 бит, в БД SHA-256), вход Basic | сессия проверяется на каждом запросе (PostgreSQL, кэш Valkey ≤ 30 с и не дольше expires_at); logout = отзыв в кэше и БД; лимит входа 5 неудач на username и 120 попыток на IP за 60 с, общий для реплик (Valkey) | сессия 3600 с; сертификат 90 дн. | `make test`, `make test-security`; аудит I: чужой CA и неверный hostname → ошибка проверки сертификата; `test_auth_acl.py`: logout, expiry, 429 + Retry-After |
| nginx → API-1/2 | `api-N:8080` HTTPS, mTLS; plaintext-listener нет | api-N: Root, SAN `api-N`, `api.internal` (nginx проверяет `api.internal`) | `lb.client.messenger` (Root, EKU clientAuth) | `app/serve.py`: CERT_REQUIRED (цепочка до Root) **и** DNS SAN ∈ `ALLOWED_CLIENT_IDENTITIES` **и** serial ∉ revoked; проверяется на каждом соединении, включая возобновлённые TLS-сессии | 90 дн.; отзыв ≤ 1 с для новых соединений | аудит A: без сертификата, чужой CA, `intruder`/`worker`/`api` от Root → соединение закрыто; `lb` → 200. `test_cert_rotation.py`: ротация, отзыв старого serial, просроченный |
| API/worker → PostgreSQL | `postgres:5432`, `sslmode=verify-full` | postgres: Root, SAN `postgres` | роли `app_api`, `app_worker` (SCRAM-SHA-256) | `pg_hba`: только `hostssl … scram-sha-256`, `host … reject`; права по таблицам: worker без users/messages/sessions и без DELETE, API без DDL и ALTER ROLE | пароль в OpenBao KV; ротация `bao.sh rotate-worker-db` | аудит B: plaintext, чужой CA, неверный hostname, неверный пароль → отказ; 9 проверок allow/deny прав |
| API/worker → MinIO | `minio:9000` HTTPS | minio: Root, SAN `minio` | `messenger-api`, `messenger-worker` | API: PUT/GET/DELETE `staging/*`, GET/DELETE `attachments/*`, LIST; worker: GET/DELETE `staging/*`, только PUT `attachments/*` | ключи в OpenBao KV (callback `s3.rotate`) | аудит C: чужой CA, plaintext → отказ; worker: GET/DELETE `attachments`, PUT `staging`, LIST, чужой префикс → AccessDenied; API: PUT `attachments`, создание бакета → AccessDenied |
| API → RabbitMQ (producer) | `rabbitmq:5671` AMQPS, mTLS; 5672 закрыт | rabbitmq: Root, SAN `rabbitmq` | `api.client.messenger` (EXTERNAL = CN) | permissions: configure `^$`, write `^attachments$`, read `^$` | 90 дн. | аудит D: без сертификата, чужой CA, `intruder`, `lb` → отказ; API: publish разрешён; consume, declare и default exchange → ACCESS_REFUSED |
| Worker → RabbitMQ (consumer) | то же | то же | `worker.client.messenger` | configure `^$`, write `^$`, read `^attachments\.process$` | 90 дн. | worker: consume разрешён; publish и declare → ACCESS_REFUSED |
| API/worker → Collector | `otel-collector:4318` OTLP/HTTPS, mTLS; gRPC 4317 выключен | otel-collector: Root | `api.otel.messenger` / `worker.otel.messenger` (Ingest CA) | client_ca_file = только Ingest CA: сертификаты Root-CA и Telemetry Clients CA отклоняются | 90 дн. | аудит E: без сертификата, чужой CA, `lb`/`api` (Root), `grafana-out` → отказ; api-otel и worker-otel → 200 |
| Collector → Prometheus / Loki / Tempo | `prometheus:9090` `/api/v1/otlp`, `loki:3100` `/otlp`, `tempo:4317` gRPC — все с TLS | Root, SAN по имени сервиса | `collector.out.messenger` (Telemetry Clients CA) | бэкенды доверяют только Telemetry Clients CA | 90 дн. | аудит F: без сертификата, чужой CA, `api` (Root), `api-otel` (Ingest) → отказ; collector-out, grafana-out, operator → 200 |
| Grafana → источники | те же HTTPS-порты | проверка сервера по Root | `grafana.out.messenger` | Telemetry Clients CA; к Collector и API не допускается | 90 дн. | аудит F (accept), E (grafana-out в Collector → отказ); запросы панелей через Grafana: `evidence/dashboard_check.txt` |
| API/worker → OpenBao | `openbao:8200` HTTPS | openbao: Root, SAN `openbao` | сертификат `api` / `worker` → cert auth (роль = ровно этот CN) | политика `messenger-<role>`: только `read` своего `messenger/data/<role>` и metadata | токен 15 мин, max 1 ч; renew/re-login каждые 15 с | аудит H: без сертификата, `intruder`, чужая роль, чужой CA → нет токена; чужой путь, запись, изменение политики, admin-пути → 403 |
| Оператор → OpenBao | не опубликован; `docker compose exec` + CLI по TLS на 127.0.0.1 | Root | root token и 3 доли unseal в `secrets/operator/` (0600) | у приложений нет admin-прав (аудит H: `sys/auth` → 403) | root token живёт у оператора; доли нужны при каждом рестарте OpenBao | `scripts/bao.sh status|unseal|seal` |
| Оператор → Grafana | `127.0.0.1:3000` HTTPS | grafana: Root, SAN `localhost` | admin-пароль (Docker secret) | anonymous выключен, регистрация выключена | — | аудит J: без логина → 401; plaintext → 400 |
| Оператор → MinIO admin | не опубликован, консоль выключена | — | root-ключи только у `minio-init` (bootstrap) | приложения используют только scoped-пользователей | — | аудит K (порт 9001 закрыт), C (у приложений нет createBucket) |
| Оператор → Prometheus/Loki/Tempo | `127.0.0.1:9090/3100/3200` | Root | `operator.messenger` (Telemetry Clients CA) | mTLS | 90 дн. | аудит F (operator → 200, без сертификата → отказ) |
| API → Valkey | `valkey:6379` TLS (plaintext port 0) | valkey: Root, SAN `valkey` | ACL-пользователь `api` (пароль из OpenBao) | ключи только `session:*` и `ratelimit:*`; команды get/set/del/incr/incrby/expire/ttl/ping; `default` выключен | пароль в OpenBao | аудит G: plaintext, чужой CA, без аутентификации, неверный пароль → отказ; другие ключи, FLUSHALL, CONFIG, KEYS → NOPERM |
| Replication / кластер | — | — | — | N/A: по одному узлу | — | аудит K: 25672/4369 (RabbitMQ), 8201 (OpenBao) закрыты |
## Служебные каналы (scrape / health / bootstrap)
| Канал | Где | Почему это не обход |
| --- | --- | --- |
| health API | `127.0.0.1:8081` внутри контейнера API, только `/health/live` и `/health/ready` | loopback-only; бизнес-API на нём нет (аудит K: 8081 закрыт из сети) |
| health nginx → API | `https://api-N:8080/health/ready` с LB-сертификатом | тот же mTLS, что у бизнес-трафика |
| self-метрики Collector | `127.0.0.1:8888`, сбор receiver `prometheus/self` внутри того же процесса | loopback-only; дальше идёт по mTLS-pipeline |
| Prometheus scrape | нет ни одного scrape-задания | все метрики приходят OTLP push по mTLS |
| healthcheck Loki/Tempo | TCP-connect на 127.0.0.1 | HTTP без клиентского сертификата невозможен |
| PostgreSQL local socket | `trust` только для Unix-сокета внутри контейнера | доступен только через `docker compose exec`, то есть оператору |
| Erlang distribution | `rabbit@localhost`, epmd и 25672 на 127.0.0.1 | из сети закрыто (аудит K) |
| bootstrap | `migrate`, `minio-init` — одноразовые задания с операторскими credentials | завершаются после настройки; у сервисов этих credentials нет |
## Ограничения модели
- Collector не связывает resource-атрибуты (`service.name`) с identity сертификата: `api-otel` может отправить данные с `service.name=messenger-worker`. Защищён сам факт ingest (только Ingest CA), но не подделка источника.
- В Prometheus, Loki и Tempo нет разграничения прав по identity: любой клиент Telemetry Clients CA и пишет, и читает. Least privilege обеспечен выдачей этого CA только трём identity (collector, grafana, operator).
- Отзыв клиентских сертификатов проверяется только в API (serial denylist). RabbitMQ, OpenBao и Collector отзыв не проверяют. Отозвать identity в RabbitMQ — удалить пользователя из `definitions.json`; в OpenBao — удалить роль `auth/cert/certs/<role>`.
- Уже открытые соединения повторно не проверяются: отзыв действует на новые соединения (keep-alive nginx → API — 4 с).
+244
View File
@@ -0,0 +1,244 @@
# Панели dashboard «Messenger — наблюдаемость (lab6)»
Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).
## SLO (окно 30 мин) и алерты
**Доступность (SLO 99%)** (stat, единица: доля (0..1))
— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.
```promql
messenger:slo_availability:ratio30m
```
**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1))
— Доля запросов в bucket le=0.5 s за 30 мин.
```promql
messenger:slo_latency:ratio30m
```
**Error ratio 5xx (2m)** (stat, единица: доля (0..1))
```promql
messenger:http_error_ratio:rate2m
```
**Старейшее задание** (stat, единица: секунды)
```promql
max(messenger_queue_oldest_age_seconds{queue="attachments.process"})
```
**Алерты Prometheus (firing/pending)** (alertlist)
Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).
**RPS API** (stat, единица: запросы/с)
```promql
messenger:http_requests:rate2m
```
**p95 API (5m)** (stat, единица: секунды)
```promql
messenger:http_latency_p95:rate5m
```
**Загрузка worker** (stat, единица: доля (0..1))
```promql
messenger:worker_saturation:ratio
```
**Ожидают пул БД** (stat, единица: штуки)
```promql
max(db_client_connection_pending_requests)
```
## API (messenger-api, все реплики)
**RPS по маршруту и методу** (timeseries, единица: запросы/с)
— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.
```promql
sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}}
```
**RPS по классу статуса** (timeseries, единица: запросы/с)
```promql
sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}}
```
**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды)
— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.
```promql
histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99
```
**Доля ошибок 5xx** (timeseries, единица: доля (0..1))
— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.
```promql
(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все
```
**p95 по маршруту** (timeseries, единица: секунды)
```promql
histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}}
```
**Запросы в обработке** (timeseries, единица: штуки)
```promql
sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}}
```
## Очередь и worker
**Jobs по исходу (в сек.)** (timeseries, единица: операции/с)
— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1
```promql
sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}}
```
**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды)
```promql
histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99
```
**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки)
— max() по репликам API: каждая публикует одно и то же значение из БД.
```promql
max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}}
```
**Возраст старейшей job** (timeseries, единица: секунды)
— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.
```promql
max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age
```
**Время до ready p50/p95/p99** (timeseries, единица: секунды)
— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.
```promql
histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99
```
**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки)
```promql
sum(messenger_worker_busy) # busy
```
```promql
sum(messenger_worker_capacity) # capacity
```
```promql
messenger:worker_saturation:ratio # saturation
```
## Ресурсы: пул БД, CPU, RAM
**Пул БД: used / idle / waiting** (timeseries, единица: штуки)
```promql
sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}}
```
```promql
sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting
```
```promql
sum by (service_name) (db_client_connection_max) # {{service_name}} max
```
**CPU процессов (ядра)** (timeseries, единица: доля (0..1))
— worker: self + завершённые дочерние процессы обработки изображений.
```promql
sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}}
```
```promql
rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector
```
**RSS процессов** (timeseries, единица: байты)
```promql
process_memory_usage_bytes # {{service_name}} {{instance}}
```
```promql
otelcol_process_memory_rss_bytes # otel-collector
```
**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки)
```promql
sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans
```
```promql
sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records
```
```promql
sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points
```
```promql
sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}}
```
```promql
sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans
```
```promql
sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full)
```
```promql
sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs
```
```promql
sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full)
```
```promql
sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}}
```
## Логи и трассы
**Логи API и worker (warning/error)** (logs)
```
{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"
```
**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table)
```
{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}
```
+90
View File
@@ -0,0 +1,90 @@
# Runbooks алертов (lab6)
Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana
(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой:
```sh
curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool
```
Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved).
Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в
цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID».
Все шаги ниже используют только сохранённую telemetry.
---
## MessengerApiHighErrorRate
**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`.
Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы).
4xx (неверный пароль, чужой чат, 409) в числитель не входят.
1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента.
2. Prometheus: на каком маршруте ошибки
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`.
3. Отличить БД от handler:
- пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`.
Ожидание пула > 0 → соединения заняты долгими запросами к БД;
- Tempo (Explore → Tempo → TraceQL):
`{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе
db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением
`LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность);
- если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler
(`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину).
4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error`
(500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`.
По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= "<request_id>"` → поле `trace_id`
→ кнопка «Open trace».
5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state,
wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` —
найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend(<pid>)`.
6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин).
## MessengerQueueLagGrowing
**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`.
Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API,
поэтому `max()`.
1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker».
- `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает;
- `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности);
- много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`.
2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id="<id>"}`. Если в трассе есть
PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал.
Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог
`outbox_relay_paused` (брокер недоступен).
3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие
`job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry.
4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование
`docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди.
5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с.
## MessengerApiLatencyHigh
**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3:
по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время
(db / S3 / handler).
## MessengerDbPoolWaiting
**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или
блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`).
## MessengerWorkerSaturated
**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker;
проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее).
## TelemetryExportFailing
**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`.
Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди.
1. `docker compose ps tempo loki prometheus` — какой backend недоступен.
2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и
`otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено.
3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются —
их объём виден в счётчиках и фиксируется в отчёте инцидента.
+11
View File
@@ -0,0 +1,11 @@
10:06:50 PASS acl_outsider eve: 404 on chat/history/metadata/content/variant/post, cannot attach a foreign file
10:06:50 PASS identity_spoofing forged X-User-Id ignored; without Bearer 401; Basic on business endpoint 401
10:06:50 PASS logout revoked token: 401 on ['api-1', 'api-2']
10:06:50 session expires_at moved to the past in PostgreSQL; the Valkey cache entry lives at most 30 s
10:07:22 PASS expiry expired session: 401 on ['api-1', 'api-2'] after the 30 s cache bound
10:07:23 PASS rehash stored m=8192,t=1,p=1 -> m=19456,t=2,p=1 after a successful login
10:07:23 PASS size_limits JSON 70 kB -> 413 (payload_too_large), upload > 10 MiB -> 413
10:07:23 PASS rate_limit_user 5 failures on ['api-1', 'api-2'] -> 429 Retry-After=60s even with the right password
10:07:24 PASS rate_limit_ip per-IP limit: 429 after 107 attempts from one address
10:07:24 waiting for the rate-limit window to expire so that later tests are not affected
PASS: acl_outsider, identity_spoofing, logout, expiry, rehash, size_limits, rate_limit_user, rate_limit_ip
+8
View File
@@ -0,0 +1,8 @@
11:08:38 openbao: sealed
11:08:40 sealed: running services keep working -> PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
11:09:00 refresher: openbao_unavailable ... keeping cached secret version (seen in API/worker logs)
11:09:35 worker-2 restarted while sealed: health=starting, waiting for OpenBao=True
11:09:36 upload during the seal processed by the remaining worker: ready
11:09:37 openbao: unsealed with 2 of 3 operator shares
11:09:52 worker-2 healthy 16 s after unseal (sealed for 58 s in total)
PASS: sealed OpenBao -> running services continue, new process waits, recovery after unseal
+11
View File
@@ -0,0 +1,11 @@
10:12:23 current lb-client serial 0E30F0061EA3866362334D7C34B7F93B9BD61418
10:12:24 issued lb-client (lb.client.messenger) serial=0E30F0061EA3866362334D7C34B7F93B9BD6142A valid 90d
10:12:25 nginx reloaded with the new client certificate (1.4 s after issue)
10:12:25 before revocation: old=HTTP/1.1 200 OK new=HTTP/1.1 200 OK
10:12:25 revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418
10:12:28 api-1 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response
10:12:29 api-2 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response
10:12:29 API log: mtls_client_rejected reason=revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418
10:12:34 traffic during rotation: 161 requests, 161 ok, unexpected errors 0
10:12:35 PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
PASS: lb-client rotated (0E30F0061EA3866362334D7C34B7F93B9BD61418 -> 0E30F0061EA3866362334D7C34B7F93B9BD6142A) without image rebuild, old serial revoked, expired cert refused
+92
View File
@@ -0,0 +1,92 @@
ok 1 series Доступность (SLO 99%) Доступность (SLO 99%)
ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%)
ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m)
ok 1 series Старейшее задание Старейшее задание
ok 1 series RPS API RPS API
ok 1 series p95 API (5m) p95 API (5m)
ok 1 series Загрузка worker Загрузка worker
ok 1 series Ожидают пул БД Ожидают пул БД
ok 12 series RPS по маршруту и методу {{http_request_method}} {{http_route}}
ok 3 series RPS по классу статуса {{status_class}}
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99
ok 1 series Доля ошибок 5xx 5xx / все
ok 10 series p95 по маршруту {{http_route}}
ok 2 series Запросы в обработке {{instance}}
ok 6 series Jobs по исходу (в сек.) {{outcome}}
ok 1 series Длительность job (succeeded) p50/p95/p99 p50
ok 1 series Длительность job (succeeded) p50/p95/p99 p95
ok 1 series Длительность job (succeeded) p50/p95/p99 p99
ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}}
ok 1 series Возраст старейшей job oldest age
ok 1 series Время до ready p50/p95/p99 p50
ok 1 series Время до ready p50/p95/p99 p95
ok 1 series Время до ready p50/p95/p99 p99
ok 1 series Насыщение worker: занято / ёмкость busy
ok 1 series Насыщение worker: занято / ёмкость capacity
ok 1 series Насыщение worker: занято / ёмкость saturation
ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}}
ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting
ok 2 series Пул БД: used / idle / waiting {{service_name}} max
ok 4 series CPU процессов (ядра) {{service_name}} {{instance}}
ok 1 series CPU процессов (ядра) otel-collector
ok 4 series RSS процессов {{service_name}} {{instance}}
ok 1 series RSS процессов otel-collector
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points
ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}}
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full)
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full)
ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}}
panels: 32, prometheus queries without data: 0
grafana prometheus ok 84 values Доступность (SLO 99%)
grafana prometheus ok 84 values Латентность ≤500 ms (SLO 95%)
grafana prometheus ok 84 values Error ratio 5xx (2m)
grafana prometheus ok 86 values Старейшее задание
grafana prometheus ok 84 values RPS API
grafana prometheus ok 84 values p95 API (5m)
grafana prometheus ok 86 values Загрузка worker
grafana prometheus ok 86 values Ожидают пул БД
grafana prometheus ok 789 values RPS по маршруту и методу
grafana prometheus ok 179 values RPS по классу статуса
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 82 values Доля ошибок 5xx
grafana prometheus ok 683 values p95 по маршруту
grafana prometheus ok 170 values Запросы в обработке
grafana prometheus ok 273 values Jobs по исходу (в сек.)
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 258 values Глубина очереди (из PostgreSQL)
grafana prometheus ok 86 values Возраст старейшей job
grafana prometheus ok 77 values Время до ready p50/p95/p99
grafana prometheus ok 77 values Время до ready p50/p95/p99
grafana prometheus ok 77 values Время до ready p50/p95/p99
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
grafana prometheus ok 344 values Пул БД: used / idle / waiting
grafana prometheus ok 172 values Пул БД: used / idle / waiting
grafana prometheus ok 172 values Пул БД: used / idle / waiting
grafana prometheus ok 350 values CPU процессов (ядра)
grafana prometheus ok 91 values CPU процессов (ядра)
grafana prometheus ok 402 values RSS процессов
grafana prometheus ok 105 values RSS процессов
grafana prometheus ok 76 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 80 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 247 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 273 values Telemetry pipeline: принято / отправлено / ошибк
grafana loki ok 100 values Логи API и worker (warning/error)
grafana tempo ok 20 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy)
via Grafana: queries without data or with errors: 0
+13
View File
@@ -0,0 +1,13 @@
# 2026-11-11T08:10:20Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.265}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab7-api-1-1 Up 38 seconds (healthy)
$ docker compose start api-1 api-2
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.672}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab7-api-1-1 Up About a minute (healthy)
$ docker compose start api-1 api-2
+342
View File
@@ -0,0 +1,342 @@
[
{
"container": "api-1-1",
"user": "10001:10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 0.5,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "api-2-1",
"user": "10001:10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 0.5,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "grafana-1",
"user": "472",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 0.5,
"memory_mib": 256,
"pids_limit": 256,
"docker_socket": false,
"published": [
"127.0.0.1:3000->3000/tcp"
],
"problems": []
},
{
"container": "loki-1",
"user": "10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [],
"cpus": 1.0,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [
"127.0.0.1:3100->3100/tcp"
],
"problems": []
},
{
"container": "minio-1",
"user": "65532",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 0.5,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "nginx-1",
"user": "101:101",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/etc/nginx/upstream",
"/tmp",
"/var/cache/nginx",
"/var/run"
],
"cpus": 0.25,
"memory_mib": 64,
"pids_limit": 256,
"docker_socket": false,
"published": [
"127.0.0.1:8443->8443/tcp"
],
"problems": []
},
{
"container": "openbao-1",
"user": "100:1000",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 0.5,
"memory_mib": 256,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "otel-collector-1",
"user": "10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [],
"cpus": 1.0,
"memory_mib": 256,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "postgres-1",
"user": "70:70",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp",
"/var/run/postgresql"
],
"cpus": 1.0,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "prometheus-1",
"user": "nobody",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [],
"cpus": 1.0,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [
"127.0.0.1:9090->9090/tcp"
],
"problems": []
},
{
"container": "rabbitmq-1",
"user": "100:101",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/etc/rabbitmq/conf.d",
"/tmp"
],
"cpus": 1.0,
"memory_mib": 512,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "tempo-1",
"user": "10001:10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 1.0,
"memory_mib": 1024,
"pids_limit": 256,
"docker_socket": false,
"published": [
"127.0.0.1:3200->3200/tcp"
],
"problems": []
},
{
"container": "valkey-1",
"user": "999:1000",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/data",
"/tmp"
],
"cpus": 0.25,
"memory_mib": 128,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "worker-1",
"user": "10001:10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 2.0,
"memory_mib": 1536,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
},
{
"container": "worker-2",
"user": "10001:10001",
"privileged": false,
"network": "mtusi-lab7_default",
"pid": "private",
"cap_drop": [
"ALL"
],
"cap_add": null,
"no_new_privileges": true,
"read_only_rootfs": true,
"tmpfs": [
"/tmp"
],
"cpus": 2.0,
"memory_mib": 1536,
"pids_limit": 256,
"docker_socket": false,
"published": [],
"problems": []
}
]
+18
View File
@@ -0,0 +1,18 @@
container user capDrop nnp ro cpus mem/pids published problems
api-1-1 10001:10001 ALL True True 0.5 512/256 - ok
api-2-1 10001:10001 ALL True True 0.5 512/256 - ok
grafana-1 472 ALL True True 0.5 256/256 127.0.0.1:3000 ok
loki-1 10001 ALL True True 1.0 512/256 127.0.0.1:3100 ok
minio-1 65532 ALL True True 0.5 512/256 - ok
nginx-1 101:101 ALL True True 0.25 64/256 127.0.0.1:8443 ok
openbao-1 100:1000 ALL True True 0.5 256/256 - ok
otel-collector-1 10001 ALL True True 1.0 256/256 - ok
postgres-1 70:70 ALL True True 1.0 512/256 - ok
prometheus-1 nobody ALL True True 1.0 512/256 127.0.0.1:9090 ok
rabbitmq-1 100:101 ALL True True 1.0 512/256 - ok
tempo-1 10001:10001 ALL True True 1.0 1024/256 127.0.0.1:3200 ok
valkey-1 999:1000 ALL True True 0.25 128/256 - ok
worker-1 10001:10001 ALL True True 2.0 1536/256 - ok
worker-2 10001:10001 ALL True True 2.0 1536/256 - ok
published ports allowed: {'8443': 'nginx business ingress (TLS)', '3000': 'Grafana (HTTPS + login)', '9090': 'Prometheus (mTLS operator)', '3100': 'Loki (mTLS operator)', '3200': 'Tempo (mTLS operator)'}
15 containers, 0 problems
+16
View File
@@ -0,0 +1,16 @@
Trivy 0.58.1, DB 2026-11-11; scanned with docker save (no Docker socket in the scanner)
== app BEFORE (python 3.12.8, pillow 11.0.0, urllib3 2.3.0)
/scan/app.tar (debian 12.9) os-pkgs {'CRITICAL': 9, 'HIGH': 145, 'LOW': 158, 'MEDIUM': 217} | HIGH+CRIT with fix: 56/154 | secrets: 0
Python lang-pkgs {'CRITICAL': 5, 'HIGH': 43, 'LOW': 2, 'MEDIUM': 26} | HIGH+CRIT with fix: 48/48 | secrets: 0
== app AFTER (python 3.12.16, pillow 13.0.0, urllib3 2.8.0)
/scan/app.tar (debian 12.15) os-pkgs {'CRITICAL': 2, 'HIGH': 96, 'LOW': 108, 'MEDIUM': 129, 'UNKNOWN': 10} | HIGH+CRIT with fix: 0/98 | secrets: 0
Python lang-pkgs {'HIGH': 3, 'LOW': 2, 'MEDIUM': 9} | HIGH+CRIT with fix: 3/3 | secrets: 0
== nginx:1.27.3-alpine
/scan/nginx.tar (alpine 3.20.5) os-pkgs {'CRITICAL': 3, 'HIGH': 37, 'LOW': 31, 'MEDIUM': 33} | HIGH+CRIT with fix: 40/40 | secrets: 0
== openbao/openbao:2.1.0
/scan/openbao.tar (alpine 3.20.3) os-pkgs {'CRITICAL': 2, 'HIGH': 26, 'LOW': 23, 'MEDIUM': 14} | HIGH+CRIT with fix: 28/28 | secrets: 0
bin/bao lang-pkgs {'CRITICAL': 2, 'HIGH': 61, 'LOW': 9, 'MEDIUM': 68, 'UNKNOWN': 2} | HIGH+CRIT with fix: 59/63 | secrets: 0
+80
View File
@@ -0,0 +1,80 @@
10:18:35 scenario A: scenario_a
10:18:35 client saw: missing X-Request-Id
10:18:35 committed attachment a3900cbb-c50d-496c-9962-ba0ec39928d3 survived the API crash
10:18:35 statuses: processing -> ready
10:18:35 audit attempt=1 enqueued api:api-1
10:18:35 audit attempt=1 claimed 2e84bfa8176e:4f81b283
10:18:35 audit attempt=1 succeeded 2e84bfa8176e:4f81b283
10:18:36 scenario A PASS (1.9s)
10:18:36 scenario B: scenario_b
10:18:59 statuses: queued -> processing -> queued -> ready
10:18:59 audit attempt=1 enqueued api:api-2
10:18:59 audit attempt=1 claimed eb9974d4233a:3bafe435
10:18:59 audit attempt=1 retry_scheduled sweeper:2e84bfa8176e:4f81b28 lease_expired delay=1.34s
10:18:59 audit attempt=2 claimed 2e84bfa8176e:4f81b283
10:18:59 audit attempt=2 succeeded 2e84bfa8176e:4f81b283
10:19:00 scenario B PASS (23.6s)
10:19:00 scenario C: scenario_c
10:19:01 statuses: queued -> ready
10:19:01 audit attempt=1 enqueued api:api-2
10:19:01 audit attempt=1 claimed eb9974d4233a:f07c7c15
10:19:01 audit attempt=1 succeeded eb9974d4233a:f07c7c15
10:19:11 scenario C PASS (10.8s)
10:19:11 scenario D: scenario_d
10:19:22 statuses: processing -> queued -> ready
10:19:22 audit attempt=1 enqueued api:api-1
10:19:22 audit attempt=1 claimed 2e84bfa8176e:4f81b283
10:19:22 audit attempt=1 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=1.33s
10:19:22 audit attempt=2 claimed eb9974d4233a:ea4f119d
10:19:22 audit attempt=2 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=2.5s
10:19:22 audit attempt=3 claimed 2e84bfa8176e:4f81b283
10:19:22 audit attempt=3 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=5.78s
10:19:22 audit attempt=4 claimed eb9974d4233a:ea4f119d
10:19:22 audit attempt=4 succeeded eb9974d4233a:ea4f119d
10:19:24 backoff delays: [1.33, 2.5, 5.78]
10:19:24 scenario D PASS (12.6s)
10:19:24 scenario E: scenario_e
10:19:24 audit attempt=1 enqueued api:api-2
10:19:24 audit attempt=1 claimed 2e84bfa8176e:4f81b283
10:19:24 audit attempt=1 failed 2e84bfa8176e:4f81b283 invalid_image
10:19:24 scenario E PASS (0.7s)
10:19:24 scenario F: scenario_f
10:19:48 failed registry contains the job; replay
10:20:11 statuses: queued -> ready
10:20:11 audit attempt=1 enqueued api:api-1
10:20:11 audit attempt=1 claimed eb9974d4233a:ea4f119d
10:20:11 audit attempt=1 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=1.28s
10:20:11 audit attempt=2 claimed 2e84bfa8176e:4f81b283
10:20:11 audit attempt=2 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=2.55s
10:20:11 audit attempt=3 claimed eb9974d4233a:ea4f119d
10:20:11 audit attempt=3 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=6.47s
10:20:11 audit attempt=4 claimed 2e84bfa8176e:4f81b283
10:20:11 audit attempt=4 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=9.94s
10:20:11 audit attempt=5 claimed eb9974d4233a:ea4f119d
10:20:11 audit attempt=5 failed eb9974d4233a:ea4f119d retry_exhausted
10:20:11 audit attempt=6 replayed operator:test-suite
10:20:11 audit attempt=6 claimed 2e84bfa8176e:4f81b283
10:20:11 audit attempt=6 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=20.59s
10:20:11 audit attempt=7 claimed eb9974d4233a:ea4f119d
10:20:11 audit attempt=7 succeeded eb9974d4233a:ea4f119d
10:20:13 second replay refused: job is succeeded, attachment is ready
10:20:14 scenario F PASS (50.2s)
10:20:14 scenario G: scenario_g
10:20:38 statuses: processing -> queued -> processing -> ready
10:20:38 audit attempt=1 enqueued api:api-1
10:20:38 audit attempt=1 claimed 2e84bfa8176e:4f81b283
10:20:38 audit attempt=1 retry_scheduled sweeper:eb9974d4233a:ea4f119 lease_expired delay=1.12s
10:20:38 audit attempt=2 claimed eb9974d4233a:ea4f119d
10:20:38 audit attempt=2 succeeded eb9974d4233a:ea4f119d
10:21:11 scenario G PASS (56.4s)
10:21:11 scenario H: scenario_h
10:21:15 DLQ depth 0 -> 1
10:21:15 scenario H PASS (4.1s)
10:21:15 scenario I: scenario_i
10:21:19 broker down: upload 202, outbox holds 1 unpublished job
10:21:25 statuses: queued -> ready
10:21:25 audit attempt=1 enqueued api:api-1
10:21:25 audit attempt=1 claimed 2e84bfa8176e:4f81b283
10:21:25 audit attempt=1 succeeded 2e84bfa8176e:4f81b283
10:21:26 scenario I PASS (11.4s)
PASS: failure windows A B C D E F G H I
+14
View File
@@ -0,0 +1,14 @@
10:18:08 stop all workers
10:18:08 upload 202 queued id=59f4941d-1a35-4d70-a976-c8ed89746f38
10:18:08 message 0f104c92-c0dd-46a0-b906-f58c9fde4017 created with queued attachment
10:18:10 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0}
10:18:11 restart api-1 api-2 rabbitmq (volumes kept)
10:18:23 started before: mtusi-lab7-api-1-1 2026-11-11T07:17:26.446583425Z; mtusi-lab7-api-2-1 2026-11-11T07:17:38.07347875Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:17:27.208089551Z
10:18:23 started after: mtusi-lab7-api-1-1 2026-11-11T07:18:11.610777668Z; mtusi-lab7-api-2-1 2026-11-11T07:18:11.669354168Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:18:12.345023127Z
10:18:24 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0}
10:18:24 start workers
10:18:27 status sequence observed: queued -> ready
10:18:28 objects: ['attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/original', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/crop.jpg', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/optimized.webp', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/thumbnail.jpg']
10:18:28 redelivery: republish the finished job through the outbox (duplicate message)
10:18:34 duplicate acked as no-op: variants=3 successes=1
PASS: worker down / API+broker restart / redelivery no-op
+87
View File
@@ -0,0 +1,87 @@
{
"argon2_bench_api_container_0_5_cpu": [
{
"m_kib": 19456,
"t": 2,
"p": 1,
"p50_ms": 11.3,
"max_ms": 66.6,
"peak_rss_mib": 37.3
},
{
"m_kib": 12288,
"t": 3,
"p": 1,
"p50_ms": 9.3,
"max_ms": 62.0,
"peak_rss_mib": 37.3
},
{
"m_kib": 47104,
"t": 1,
"p": 1,
"p50_ms": 16.6,
"max_ms": 69.2,
"peak_rss_mib": 83.0
},
{
"m_kib": 65536,
"t": 3,
"p": 1,
"p50_ms": 171.6,
"max_ms": 175.5,
"peak_rss_mib": 101.0
},
{
"m_kib": 65536,
"t": 3,
"p": 4,
"p50_ms": 390.0,
"max_ms": 806.1,
"peak_rss_mib": 101.2
}
],
"baseline": {
"attackers": 0,
"seconds": 35,
"login_attempts_per_s": 0.0,
"login_status_counts": {},
"login_401_p50_ms": null,
"login_401_p99_ms": null,
"probe_ok": 165,
"probe_errors": 0,
"probe_p50_ms": 9.5,
"probe_p99_ms": 46.4,
"api_cpu_pct_of_one_core_both_replicas": 10.3,
"api_rss_samples": {
"mtusi-lab7-api-2-1": "97.59MiB",
"mtusi-lab7-api-1-1": "97.34MiB"
}
},
"flood": {
"attackers": 32,
"seconds": 37,
"login_attempts_per_s": 40.4,
"login_status_counts": {
"401": 1496
},
"login_401_p50_ms": 802.6,
"login_401_p99_ms": 2396.6,
"probe_ok": 128,
"probe_errors": 0,
"probe_p50_ms": 28.5,
"probe_p99_ms": 295.0,
"api_cpu_pct_of_one_core_both_replicas": 89.6,
"api_rss_samples": {
"mtusi-lab7-api-2-1": "196.7MiB",
"mtusi-lab7-api-1-1": "183.3MiB"
}
},
"config": {
"PASSWORD_HASH_CONCURRENCY_per_replica": 4,
"PASSWORD_HASH_QUEUE_SECONDS": 3,
"replicas": 2,
"api_cpu_limit": 0.5,
"api_mem_limit_mib": 512
}
}
+27
View File
@@ -0,0 +1,27 @@
10:32:44 message sent request_id=94e97b26265542fb9d208a905b5d5701 (text marker kept private)
10:32:47 upload 913f8afd-4a07-4a58-b943-ea623e6ec0d0: queued -> queued -> processing -> ready
10:32:51 Tempo: trace 715b953a7f7536b44b99c48205a5c639, 42 spans; tree (DB client spans omitted):
api SERVER POST /api/v1/chats/{chat_id}/attachments 15.3 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "95e2a7eb2f9e46319830eca2e087ccfb", "messenger.instance_id": "api-2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
api CLIENT S3.PutObject 8.4 ms UNSET {"rpc.method": "PutObject"}
api PRODUCER attachments.process publish 3.1 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "1", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
worker CONSUMER attachments.process process 8.3 ms ERROR {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"}
api PRODUCER attachments.process publish 4.8 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
worker CONSUMER attachments.process process 174.7 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"}
worker CLIENT S3.GetObject 15.2 ms UNSET {"rpc.method": "GetObject"}
worker INTERNAL imageproc 128.0 ms UNSET {"messenger.pipeline_version": "1"}
worker CLIENT S3.CopyObject 5.3 ms UNSET {"rpc.method": "CopyObject"}
worker CLIENT S3.PutObject 3.0 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.PutObject 2.1 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"}
10:32:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links
10:32:51 Loki: 4 records with trace_id=715b953a7f7536b44b99c48205a5c639: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished')]
10:32:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts']
10:32:55 HTTP 500 X-Request-Id=2df37bc19c3d4676804688b4abcb2ae1 -> Loki event=unhandled_error trace_id=f63428fc5f86c4f25be261c5db8ecc08
10:32:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True
10:32:55 message POST trace 3157b85474b226259a91fb7e6ddeec5a: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys']
10:32:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs
10:32:55 PASS: 1921 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_method', 'http_request_method', 'http_response_status_code', 'http_route', 'http_scheme', 'http_status_code', 'instance', 'job', 'job_type', 'le', 'net_host_name', 'net_host_port', 'net_protocol_name', 'net_protocol_version', 'network_protocol_version', 'otel_scope_name', 'otel_scope_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme']
10:32:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me']
10:33:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
+36
View File
@@ -0,0 +1,36 @@
# 2026-11-11T08:09:52Z scripts/restart.sh
$ docker compose ps (before)
mtusi-lab7-api-1-1 Up 51 minutes (healthy)
mtusi-lab7-api-2-1 Up 51 minutes (healthy)
mtusi-lab7-grafana-1 Up About an hour (healthy)
mtusi-lab7-loki-1 Up About an hour (healthy)
mtusi-lab7-minio-1 Up About an hour (healthy)
mtusi-lab7-nginx-1 Up 43 minutes (healthy)
mtusi-lab7-openbao-1 Up About a minute (healthy)
mtusi-lab7-otel-collector-1 Up 42 minutes
mtusi-lab7-postgres-1 Up About an hour (healthy)
mtusi-lab7-prometheus-1 Up About an hour (healthy)
mtusi-lab7-rabbitmq-1 Up 40 minutes (healthy)
mtusi-lab7-tempo-1 Up About an hour (healthy)
mtusi-lab7-valkey-1 Up About an hour (healthy)
mtusi-lab7-worker-1 Up 50 minutes (healthy)
mtusi-lab7-worker-2 Up 51 seconds (healthy)
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
$ docker compose ps (after)
mtusi-lab7-api-1-1 Up 2 seconds (health: starting)
mtusi-lab7-api-2-1 Up 1 second (health: starting)
mtusi-lab7-grafana-1 Up 4 seconds (health: starting)
mtusi-lab7-loki-1 Up 2 seconds (health: starting)
mtusi-lab7-minio-1 Up 7 seconds (healthy)
mtusi-lab7-nginx-1 Up 2 seconds (health: starting)
mtusi-lab7-openbao-1 Up 7 seconds (healthy)
mtusi-lab7-otel-collector-1 Up 4 seconds
mtusi-lab7-postgres-1 Up 7 seconds (healthy)
mtusi-lab7-prometheus-1 Up 4 seconds (health: starting)
mtusi-lab7-rabbitmq-1 Up 6 seconds (healthy)
mtusi-lab7-tempo-1 Up 4 seconds (health: starting)
mtusi-lab7-valkey-1 Up 7 seconds (healthy)
mtusi-lab7-worker-1 Up 1 second (health: starting)
mtusi-lab7-worker-2 Up 1 second (health: starting)
+8
View File
@@ -0,0 +1,8 @@
11:08:20 before: old password accepted, 2 app_worker connections open
11:08:21 rotated: app_worker password changed in PostgreSQL and OpenBao (messenger/worker, new KV version)
11:08:30 both workers loaded the new KV version and switched the pool (10 s after rotation)
11:08:30 terminated all existing app_worker sessions: the pools must reconnect with the new password
11:08:36 3 uploads after the forced reconnect are ready; 2 app_worker connections open again
11:08:37 probe: old password -> rejected: FATAL: password authentication failed for user "app_worker" | new password -> accepted
11:08:37 traffic: 116 requests, unexpected errors 0, uploads 20; failed jobs since rotation: 0
PASS: app_worker password rotated via OpenBao without rebuild/restart; old password revoked
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
@@ -0,0 +1,9 @@
# 2026-11-11T08:08:18Z (OpenBao sealed at this moment: running services use the secrets they already hold)
$ make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make validate
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
$ make test CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
$ make test-security CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
File diff suppressed because it is too large Load Diff
+124
View File
@@ -0,0 +1,124 @@
PASS A LB -> api-1:8080 no client cert expect=reject rejected: closed without response
PASS A LB -> api-1:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity intruder expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity worker expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity api expect=reject rejected: closed without response
PASS A LB -> api-1:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
PASS A LB -> api-1:8080 plaintext client expect=reject server answered b''
PASS A LB -> api-2:8080 no client cert expect=reject rejected: closed without response
PASS A LB -> api-2:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity intruder expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity worker expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity api expect=reject rejected: closed without response
PASS A LB -> api-2:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
PASS A LB -> api-2:8080 plaintext client expect=reject server answered b''
PASS I client -> nginx:8443 trusted CA, hostname nginx expect=accept accepted: HTTP/1.1 200 OK
PASS I client -> nginx:8443 untrusted CA in client trust store expect=reject rejected: server cert verify failed: unable to get local issuer certificate
PASS I client -> nginx:8443 wrong hostname (evil.example) expect=reject rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'.
PASS I client -> nginx:8443 plaintext client expect=reject server answered b'HTTP/1.1 400 Bad Request'
PASS B svc -> postgres:5432 plaintext (sslmode=disable) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: pg_hba.c
PASS B svc -> postgres:5432 foreign CA as root (verify-full) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: SSL error: certi
PASS B svc -> postgres:5432 hostname mismatch (IP, verify-full) expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: server certificate for "postgres"
PASS B svc -> postgres:5432 TLS verify-full + app_api password expect=accept connected
PASS B svc -> postgres:5432 wrong password expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: password authentication fa
PASS B app_worker privileges SELECT count(*) FROM users expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM messages expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM sessions expect=deny InsufficientPrivilege
PASS B app_worker privileges DELETE FROM attachments WHERE false expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM attachment_jobs expect=allow executed
PASS B app_api privileges SELECT count(*) FROM users expect=allow executed
PASS B app_api privileges CREATE TABLE audit_probe (x int) expect=deny InsufficientPrivilege
PASS B app_api privileges ALTER ROLE app_worker PASSWORD 'x' expect=deny InsufficientPrivilege
PASS B app_api privileges SELECT count(*) FROM schema_migrations expect=deny InsufficientPrivilege
PASS C svc -> minio:9000 foreign CA as trust anchor expect=reject SSLError
PASS C svc -> minio:9000 plaintext http:// expect=reject ConnectionClosedError
PASS C worker S3 rights PUT attachments/* expect=allow ok
PASS C worker S3 rights GET attachments/* expect=deny AccessDenied
PASS C worker S3 rights DELETE attachments/* expect=deny AccessDenied
PASS C worker S3 rights PUT staging/* expect=deny AccessDenied
PASS C worker S3 rights LIST bucket expect=deny AccessDenied
PASS C worker S3 rights PUT outside prefixes expect=deny AccessDenied
PASS C api S3 rights PUT staging/* expect=allow ok
PASS C api S3 rights PUT attachments/* (only worker publishes) expect=deny AccessDenied
PASS C api S3 rights GET attachments/* expect=allow ok
PASS C api S3 rights LIST bucket (reconciler) expect=allow ok
PASS C api S3 rights create bucket expect=deny AccessDenied
PASS C api S3 rights cleanup audit objects expect=allow ok
PASS D svc -> rabbitmq:5672 plaintext client expect=reject connection failed: ConnectionRefusedError
PASS D svc -> rabbitmq:5671 no client cert expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_
PASS D svc -> rabbitmq:5671 foreign CA (claims worker name) expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U
PASS D svc -> rabbitmq:5671 same CA, unknown identity intruder expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
PASS D svc -> rabbitmq:5671 same CA, identity lb expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
PASS D api AMQP rights publish to exchange attachments expect=allow ok
PASS D api AMQP rights consume attachments.process expect=deny ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost '
PASS D api AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
PASS D api AMQP rights publish via default exchange expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess
PASS D worker AMQP rights consume attachments.process expect=allow ok
PASS D worker AMQP rights publish to exchange attachments expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess
PASS D worker AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
PASS E svc -> otel-collector:4318 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS E svc -> otel-collector:4318 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 root CA identity lb (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 root CA identity api (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 telemetry CA (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 ingest CA api.otel expect=accept accepted: HTTP/1.1 200 OK
PASS E svc -> otel-collector:4318 ingest CA worker.otel expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> prometheus:9090 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
PASS F -> loki:3100 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> loki:3100 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> loki:3100 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> loki:3100 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
PASS F -> tempo:3200 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> tempo:3200 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> tempo:3200 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> tempo:3200 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
PASS F collector -> tempo:4317 (gRPC) no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F collector -> tempo:4317 (gRPC) ingest CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F collector -> tempo:4317 (gRPC) telemetry CA collector.out expect=accept accepted: @
PASS G api -> valkey:6379 plaintext client expect=reject connection failed: ConnectionResetError
PASS G api -> valkey:6379 foreign CA as trust anchor expect=deny ConnectionError Error 1 connecting to valkey:6379. [SSL: CERTIFICATE_VERIFY_
PASS G api -> valkey:6379 no authentication (default user off) expect=deny AuthenticationError Authentication required.
PASS G api -> valkey:6379 user api, wrong password expect=deny AuthenticationError invalid username-password pair or user is disabled.
PASS G api -> valkey:6379 api: SET/GET session:* expect=allow ok
PASS G api -> valkey:6379 api: INCR ratelimit:* expect=allow ok
PASS G api -> valkey:6379 api: GET outside allowed keys expect=deny NoPermissionError No permissions to access a key
PASS G api -> valkey:6379 api: FLUSHALL expect=deny NoPermissionError User api has no permissions to run the 'flushall' command
PASS G api -> valkey:6379 api: CONFIG GET expect=deny NoPermissionError User api has no permissions to run the 'config|get' command
PASS G api -> valkey:6379 api: KEYS * expect=deny NoPermissionError User api has no permissions to run the 'keys' command
PASS G api -> valkey:6379 health: GET session:* expect=deny NoPermissionError User health has no permissions to run the 'get' command
PASS H svc -> openbao:8200 cert login: no client cert expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: same CA, identity intruder expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: worker cert, role api expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: api cert, role worker expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: lb cert, role api expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: foreign CA (claims worker name) expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: worker cert, role worker expect=accept HTTP 200 token=True
PASS H worker token policies / ttl expect=scoped policies=['default', 'messenger-worker'] ttl=900s
PASS H worker token read own messenger/worker expect=allow HTTP 200
PASS H worker token read messenger/api expect=deny HTTP 403
PASS H worker token write own secret expect=deny HTTP 403
PASS H worker token write own policy expect=deny HTTP 403
PASS H worker token list auth methods (admin) expect=deny HTTP 403
PASS H svc -> openbao:8200 cert login: api cert, role api expect=accept HTTP 200 token=True
PASS H api token policies / ttl expect=scoped policies=['default', 'messenger-api'] ttl=900s
PASS H api token read own messenger/api expect=allow HTTP 200
PASS H api token read messenger/worker expect=deny HTTP 403
PASS H api token write own secret expect=deny HTTP 403
PASS H api token write own policy expect=deny HTTP 403
PASS H api token list auth methods (admin) expect=deny HTTP 403
PASS J operator -> grafana:3000 HTTPS without login expect=reject accepted: HTTP/1.1 401 Unauthorized
PASS J operator -> grafana:3000 plaintext client expect=reject server answered b'HTTP/1.0 400 Bad Request'
122 checks, 0 failed
+139
View File
@@ -0,0 +1,139 @@
Container mtusi-lab7-worker-run-dcda01946f3d Creating
Container mtusi-lab7-worker-run-dcda01946f3d Created
PASS A LB -> api-1:8080 no client cert expect=reject rejected: closed without response
PASS A LB -> api-1:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity intruder expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity worker expect=reject rejected: closed without response
PASS A LB -> api-1:8080 same CA, identity api expect=reject rejected: closed without response
PASS A LB -> api-1:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
PASS A LB -> api-1:8080 plaintext client expect=reject server answered b''
PASS A LB -> api-2:8080 no client cert expect=reject rejected: closed without response
PASS A LB -> api-2:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity intruder expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity worker expect=reject rejected: closed without response
PASS A LB -> api-2:8080 same CA, identity api expect=reject rejected: closed without response
PASS A LB -> api-2:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
PASS A LB -> api-2:8080 plaintext client expect=reject server answered b''
PASS I client -> nginx:8443 trusted CA, hostname nginx expect=accept accepted: HTTP/1.1 200 OK
PASS I client -> nginx:8443 untrusted CA in client trust store expect=reject rejected: server cert verify failed: unable to get local issuer certificate
PASS I client -> nginx:8443 wrong hostname (evil.example) expect=reject rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'.
PASS I client -> nginx:8443 plaintext client expect=reject server answered b'HTTP/1.1 400 Bad Request'
PASS B svc -> postgres:5432 plaintext (sslmode=disable) expect=reject OperationalError: connection failed: connection to server at "172.22.0.9", port 5432 failed: FATAL: pg_hba.co
PASS B svc -> postgres:5432 foreign CA as root (verify-full) expect=reject OperationalError: connection failed: connection to server at "172.22.0.9", port 5432 failed: SSL error: certif
PASS B svc -> postgres:5432 hostname mismatch (IP, verify-full) expect=reject connection failed: connection to server at "172.22.0.9", port 5432 failed: server certificate for "postgres" (
PASS B svc -> postgres:5432 TLS verify-full + app_api password expect=accept connected
PASS B svc -> postgres:5432 wrong password expect=reject connection failed: connection to server at "172.22.0.9", port 5432 failed: FATAL: password authentication fai
PASS B app_worker privileges SELECT count(*) FROM users expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM messages expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM sessions expect=deny InsufficientPrivilege
PASS B app_worker privileges DELETE FROM attachments WHERE false expect=deny InsufficientPrivilege
PASS B app_worker privileges SELECT count(*) FROM attachment_jobs expect=allow executed
PASS B app_api privileges SELECT count(*) FROM users expect=allow executed
PASS B app_api privileges CREATE TABLE audit_probe (x int) expect=deny InsufficientPrivilege
PASS B app_api privileges ALTER ROLE app_worker PASSWORD 'x' expect=deny InsufficientPrivilege
PASS B app_api privileges SELECT count(*) FROM schema_migrations expect=deny InsufficientPrivilege
PASS C svc -> minio:9000 foreign CA as trust anchor expect=reject SSLError
PASS C svc -> minio:9000 plaintext http:// expect=reject ConnectionClosedError
PASS C worker S3 rights PUT attachments/* expect=allow ok
PASS C worker S3 rights GET attachments/* expect=deny AccessDenied
PASS C worker S3 rights DELETE attachments/* expect=deny AccessDenied
PASS C worker S3 rights PUT staging/* expect=deny AccessDenied
PASS C worker S3 rights LIST bucket expect=deny AccessDenied
PASS C worker S3 rights PUT outside prefixes expect=deny AccessDenied
PASS C api S3 rights PUT staging/* expect=allow ok
PASS C api S3 rights PUT attachments/* (only worker publishes) expect=deny AccessDenied
PASS C api S3 rights GET attachments/* expect=allow ok
PASS C api S3 rights LIST bucket (reconciler) expect=allow ok
PASS C api S3 rights create bucket expect=deny AccessDenied
PASS C api S3 rights cleanup audit objects expect=allow ok
PASS D svc -> rabbitmq:5672 plaintext client expect=reject connection failed: ConnectionRefusedError
PASS D svc -> rabbitmq:5671 no client cert expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_
PASS D svc -> rabbitmq:5671 foreign CA (claims worker name) expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U
PASS D svc -> rabbitmq:5671 same CA, unknown identity intruder expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
PASS D svc -> rabbitmq:5671 same CA, identity lb expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
PASS D api AMQP rights publish to exchange attachments expect=allow ok
PASS D api AMQP rights consume attachments.process expect=deny ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost '
PASS D api AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
PASS D api AMQP rights publish via default exchange expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess
PASS D worker AMQP rights consume attachments.process expect=allow ok
PASS D worker AMQP rights publish to exchange attachments expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess
PASS D worker AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
PASS E svc -> otel-collector:4318 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS E svc -> otel-collector:4318 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 root CA identity lb (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 root CA identity api (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 telemetry CA (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS E svc -> otel-collector:4318 ingest CA api.otel expect=accept accepted: HTTP/1.1 200 OK
PASS E svc -> otel-collector:4318 ingest CA worker.otel expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> prometheus:9090 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> prometheus:9090 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
PASS F -> prometheus:9090 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
PASS F -> loki:3100 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> loki:3100 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> loki:3100 telemetry CA collector.out expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F -> loki:3100 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F -> loki:3100 telemetry CA operator expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F -> tempo:3200 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F -> tempo:3200 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F -> tempo:3200 telemetry CA collector.out expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F -> tempo:3200 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F -> tempo:3200 telemetry CA operator expect=accept accepted: HTTP/1.1 503 Service Unavailable
PASS F collector -> tempo:4317 (gRPC) no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
PASS F collector -> tempo:4317 (gRPC) ingest CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
PASS F collector -> tempo:4317 (gRPC) telemetry CA collector.out expect=accept accepted: @
PASS G api -> valkey:6379 plaintext client expect=reject connection failed: ConnectionResetError
PASS G api -> valkey:6379 foreign CA as trust anchor expect=deny ConnectionError Error 1 connecting to valkey:6379. [SSL: CERTIFICATE_VERIFY_
PASS G api -> valkey:6379 no authentication (default user off) expect=deny AuthenticationError Authentication required.
PASS G api -> valkey:6379 user api, wrong password expect=deny AuthenticationError invalid username-password pair or user is disabled.
PASS G api -> valkey:6379 api: SET/GET session:* expect=allow ok
PASS G api -> valkey:6379 api: INCR ratelimit:* expect=allow ok
PASS G api -> valkey:6379 api: GET outside allowed keys expect=deny NoPermissionError No permissions to access a key
PASS G api -> valkey:6379 api: FLUSHALL expect=deny NoPermissionError User api has no permissions to run the 'flushall' command
PASS G api -> valkey:6379 api: CONFIG GET expect=deny NoPermissionError User api has no permissions to run the 'config|get' command
PASS G api -> valkey:6379 api: KEYS * expect=deny NoPermissionError User api has no permissions to run the 'keys' command
PASS G api -> valkey:6379 health: GET session:* expect=deny NoPermissionError User health has no permissions to run the 'get' command
PASS H svc -> openbao:8200 cert login: no client cert expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: same CA, identity intruder expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: worker cert, role api expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: api cert, role worker expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: lb cert, role api expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: foreign CA (claims worker name) expect=reject HTTP 400 token=False
PASS H svc -> openbao:8200 cert login: worker cert, role worker expect=accept HTTP 200 token=True
PASS H worker token policies / ttl expect=scoped policies=['default', 'messenger-worker'] ttl=900s
PASS H worker token read own messenger/worker expect=allow HTTP 200
PASS H worker token read messenger/api expect=deny HTTP 403
PASS H worker token write own secret expect=deny HTTP 403
PASS H worker token write own policy expect=deny HTTP 403
PASS H worker token list auth methods (admin) expect=deny HTTP 403
PASS H svc -> openbao:8200 cert login: api cert, role api expect=accept HTTP 200 token=True
PASS H api token policies / ttl expect=scoped policies=['default', 'messenger-api'] ttl=900s
PASS H api token read own messenger/api expect=allow HTTP 200
PASS H api token read messenger/worker expect=deny HTTP 403
PASS H api token write own secret expect=deny HTTP 403
PASS H api token write own policy expect=deny HTTP 403
PASS H api token list auth methods (admin) expect=deny HTTP 403
PASS J operator -> grafana:3000 HTTPS without login expect=reject accepted: HTTP/1.1 401 Unauthorized
PASS J operator -> grafana:3000 plaintext client expect=reject server answered b'HTTP/1.0 400 Bad Request'
PASS K rabbitmq:25672 Erlang distribution expect=closed ConnectionRefusedError
PASS K rabbitmq:4369 epmd expect=closed ConnectionRefusedError
PASS K rabbitmq:15672 management UI expect=closed ConnectionRefusedError
PASS K loki:9096 Loki internal gRPC expect=closed ConnectionRefusedError
PASS K tempo:9095 Tempo internal gRPC expect=closed ConnectionRefusedError
PASS K otel-collector:8888 collector self-metrics expect=closed ConnectionRefusedError
PASS K otel-collector:13133 collector health expect=closed ConnectionRefusedError
PASS K otel-collector:4317 collector OTLP/gRPC (disabled) expect=closed ConnectionRefusedError
PASS K api-1:8081 API loopback health expect=closed ConnectionRefusedError
PASS K api-2:8081 API loopback health expect=closed ConnectionRefusedError
PASS K openbao:8201 OpenBao cluster port expect=closed ConnectionRefusedError
PASS K minio:9001 MinIO console expect=closed ConnectionRefusedError
PASS K postgres:5433 no second PostgreSQL listener expect=closed ConnectionRefusedError
135 checks, 0 failed
+15
View File
@@ -0,0 +1,15 @@
PASS K rabbitmq:25672 Erlang distribution expect=closed ConnectionRefusedError
PASS K rabbitmq:4369 epmd expect=closed ConnectionRefusedError
PASS K rabbitmq:15672 management UI expect=closed ConnectionRefusedError
PASS K loki:9096 Loki internal gRPC expect=closed ConnectionRefusedError
PASS K tempo:9095 Tempo internal gRPC expect=closed ConnectionRefusedError
PASS K otel-collector:8888 collector self-metrics expect=closed ConnectionRefusedError
PASS K otel-collector:13133 collector health expect=closed ConnectionRefusedError
PASS K otel-collector:4317 collector OTLP/gRPC (disabled) expect=closed ConnectionRefusedError
PASS K api-1:8081 API loopback health expect=closed ConnectionRefusedError
PASS K api-2:8081 API loopback health expect=closed ConnectionRefusedError
PASS K openbao:8201 OpenBao cluster port expect=closed ConnectionRefusedError
PASS K minio:9001 MinIO console expect=closed ConnectionRefusedError
PASS K postgres:5433 no second PostgreSQL listener expect=closed ConnectionRefusedError
13 checks, 0 failed
+20
View File
@@ -0,0 +1,20 @@
ok 320x200 -> thumbnail 256x160
ok crop 128x128
ok optimized WebP 320x200, no upscale
ok thumbnail/crop are JPEG
ok 40x10: thumbnail 40x10, crop 128x128
ok 3000x2 -> 256x1 / 1280x1 (min 1px)
ok 1001x999 -> 256x255 (floor)
ok EXIF orientation applied: 400x200 (o=6) -> 128x256
ok rotated content: red half on top after orientation 6
ok EXIF removed from thumbnail/optimized
ok transparent PNG -> white JPEG thumbnail
ok WebP optimized keeps alpha
ok 16-bit PNG, CMYK JPEG, palette+transparency PNG are processed
ok pixel limit enforced before decode -> image_too_large (exit 4)
ok corrupted PNG -> invalid_image (exit 3)
ok 36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got 5
ok 39.5 Mpx PNG (31 MB file) ok in 1.0s, peak child RSS 320 MiB
ok CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got -24
ok 39.5 Mpx JPEG (7 MB file) ok in 0.3s, peak child RSS 337 MiB
RESULT: PASS
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,63 @@
# Grafana 11.5.2 datasources (fixed uids are referenced by the dashboard JSON). Grafana stores no signals itself.
# "$$" escapes "$" from Grafana's env-variable interpolation; $__file{...} reads the client TLS material.
# lab7: every datasource uses HTTPS, verifies the server against the lab root CA and authenticates with grafana-out.
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: https://prometheus:9090
isDefault: true
jsonData:
tlsAuth: true
tlsAuthWithCACert: true
timeInterval: 15s
prometheusType: Prometheus
secureJsonData:
tlsCACert: "$__file{/run/secrets/ca.crt}"
tlsClientCert: "$__file{/run/secrets/client.crt}"
tlsClientKey: "$__file{/run/secrets/client.key}"
- name: Loki
uid: loki
type: loki
access: proxy
url: https://loki:3100
jsonData:
tlsAuth: true
tlsAuthWithCACert: true
maxLines: 1000
derivedFields:
# trace_id is OTLP structured metadata -> link "Open trace" to Tempo.
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
urlDisplayLabel: Open trace
secureJsonData:
tlsCACert: "$__file{/run/secrets/ca.crt}"
tlsClientCert: "$__file{/run/secrets/client.crt}"
tlsClientKey: "$__file{/run/secrets/client.key}"
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: https://tempo:3200
jsonData:
tlsAuth: true
tlsAuthWithCACert: true
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-5m"
spanEndTimeShift: "5m"
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"'
nodeGraph:
enabled: true
secureJsonData:
tlsCACert: "$__file{/run/secrets/ca.crt}"
tlsClientCert: "$__file{/run/secrets/client.crt}"
tlsClientKey: "$__file{/run/secrets/client.key}"
+48
View File
@@ -0,0 +1,48 @@
# Loki 3.4.2 single binary: log store. Logs arrive by OTLP (https://loki:3100/otlp, mTLS) from the Collector.
# service.name -> stream label service_name; log attributes (event, code.*) and trace_id/span_id -> structured metadata.
auth_enabled: false
server:
http_listen_port: 3100
# Internal gRPC of the single binary: loopback only (closed local hop, not reachable from the network).
grpc_listen_address: 127.0.0.1
grpc_listen_port: 9096
log_level: warn
# lab7: HTTPS with mandatory client certificates from the "Telemetry Clients CA" (collector-out pushes,
# grafana-out queries, operator evidence). No certificate / another CA -> TLS handshake refused.
http_tls_config:
cert_file: /run/secrets/server.crt
key_file: /run/secrets/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/secrets/telemetry-ca.crt
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
volume_enabled: true
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Idempotent MinIO bootstrap over HTTPS (CA verified by mc). Root credentials are used only here.
# Two service users with prefix-scoped policies: messenger-api and messenger-worker (see policy-*.json).
set -eu
BUCKET="${S3_BUCKET:-messenger-attachments}"
mkdir -p "$MC_CONFIG_DIR/certs/CAs"
cp /run/secrets/ca.crt "$MC_CONFIG_DIR/certs/CAs/ca.crt"
until mc alias set local https://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null 2>&1; do
echo "waiting for minio"; sleep 2
done
mc mb --ignore-existing "local/$BUCKET" >/dev/null
mc anonymous set none "local/$BUCKET" >/dev/null
for role in api worker; do
user="$(cat /run/secrets/s3_${role}_access_key)"
mc admin policy create local "messenger-$role" "/etc/minio-policy/policy-$role.json" >/dev/null
mc admin user add local "$user" "$(cat /run/secrets/s3_${role}_secret_key)" >/dev/null
mc admin policy attach local "messenger-$role" --user "$user" >/dev/null 2>&1 || true
done
echo "minio bootstrap complete: bucket=$BUCKET private, users messenger-api / messenger-worker with prefix policies"
+23
View File
@@ -0,0 +1,23 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "UploadToStagingAndCompensate",
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]
},
{
"Sid": "DownloadAndReconcileFinalObjects",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
},
{
"Sid": "ProbeAndListForReconciler",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+17
View File
@@ -0,0 +1,17 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "ReadAndCleanStaging",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/staging/*"]
},
{
"Sid": "WriteFinalObjectsOnly",
"Effect": "Allow",
"Action": ["s3:PutObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
}
]
}
+65
View File
@@ -0,0 +1,65 @@
#!/bin/sh
set -u
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
INTERVAL="${HEALTH_INTERVAL:-1}"
FALL="${HEALTH_FALL:-2}"
RISE="${HEALTH_RISE:-2}"
OUT=/etc/nginx/upstream/servers.conf
mkdir -p "$(dirname "$OUT")"
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
render() {
any_up=0
for s in $SERVERS; do
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
done
for s in $SERVERS; do
mark=""
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
mark=" down"
fi
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
done
}
for s in $SERVERS; do
v=$(var "$s")
eval "up_$v=1 ok_$v=0 bad_$v=0"
done
render > "$OUT"
watch() {
pending=0
while sleep "$INTERVAL"; do
changed=0
for s in $SERVERS; do
v=$(var "$s")
# Readiness over the same mTLS as real traffic (LB identity, server verified against the lab CA).
if curl -sf -m 1 -o /dev/null --cacert /run/secrets/ca.crt --cert /run/secrets/client.crt \
--key /run/secrets/client.key "https://$s:8080/health/ready" 2>/dev/null; then
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
fi
else
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
fi
fi
done
if [ "$changed" = 1 ]; then
render > "$OUT"
pending=1
fi
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
pending=0
fi
done
}
watch &
exec nginx -g 'daemon off;'
+95
View File
@@ -0,0 +1,95 @@
# nginx 1.27 (lab7): the only business ingress. Client -> nginx: TLS (server cert SAN localhost/nginx).
# nginx -> every API replica: mTLS. nginx presents lb-client (identity lb.client.messenger) and verifies the replica
# certificate against the lab root CA with the shared SAN api.internal. The API itself checks the LB identity.
worker_processes 2;
pid /var/run/nginx.pid;
events {
worker_connections 256;
}
http {
resolver 127.0.0.11 valid=5s ipv6=off;
upstream api {
zone api 64k;
include /etc/nginx/upstream/servers.conf;
keepalive 32;
keepalive_timeout 4s;
}
log_format main '$remote_addr - $upstream_addr [$time_local] '
'"$request" $status $body_bytes_sent '
'"$upstream_response_time" "$request_time"';
access_log /var/log/nginx/access.log main;
server {
listen 8443 ssl;
ssl_certificate /run/secrets/server.crt;
ssl_certificate_key /run/secrets/server.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_cache shared:tls:5m;
add_header Strict-Transport-Security "max-age=31536000" always;
server_tokens off;
client_max_body_size 64k;
error_page 413 @payload_too_large;
# Upstream mTLS (no fallback to plaintext, verification always on).
proxy_ssl_certificate /run/secrets/client.crt;
proxy_ssl_certificate_key /run/secrets/client.key;
proxy_ssl_trusted_certificate /run/secrets/ca.crt;
proxy_ssl_verify on;
proxy_ssl_verify_depth 2;
proxy_ssl_name api.internal;
proxy_ssl_server_name on;
proxy_ssl_protocols TLSv1.2 TLSv1.3;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 5s;
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_pass https://api;
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
}
location ~ ^/api/v1/chats/[^/]+/attachments$ {
client_max_body_size 10m;
proxy_request_buffering off;
proxy_pass https://api;
proxy_next_upstream off;
}
location @payload_too_large {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header X-Instance-Id lb always;
add_header Connection close always;
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
}
location /health/ {
proxy_pass https://api;
proxy_next_upstream error timeout;
}
location ~ ^/api/v1/attachments/ {
proxy_max_temp_file_size 0;
proxy_pass https://api;
proxy_next_upstream error timeout;
}
location / {
proxy_pass https://api;
proxy_next_upstream error timeout http_502 http_503;
}
}
}
+39 -4
View File
@@ -1,7 +1,42 @@
# OpenBao — реализовать конфигурацию и политики
# OpenBao — конфигурация и действия оператора
Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key.
OpenBao 2.6.5 в **server mode** (не dev): хранилище `file` на volume `baodata`, TLS-listener `openbao:8200` с сертификатом от lab Root CA. Наружу порт не опубликован; оператор работает через `docker compose exec` и CLI по TLS на `127.0.0.1` внутри контейнера.
Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения.
| Файл | Назначение |
| --- | --- |
| `config.hcl` | listener (TLS ≥ 1.2, `tls_client_ca_file` — запрос клиентского сертификата для cert auth), storage `file`, `disable_mlock` |
| `policies/messenger-api.hcl` | `read` только `messenger/data/api` и `messenger/metadata/api` |
| `policies/messenger-worker.hcl` | `read` только `messenger/data/worker` и `messenger/metadata/worker` |
| `../../scripts/bao.sh` | init, unseal, bootstrap, seal, rotate-worker-db, status (ничего секретного не печатает) |
Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md.
## Аутентификация сервисов
Метод **cert**:
- роль `api`: сертификат цепочкой до lab Root CA и CN ровно `api.client.messenger`, политика `messenger-api`;
- роль `worker`: то же для `worker.client.messenger`, политика `messenger-worker`;
- у обеих ролей `token_ttl=15m`, `token_max_ttl=1h`.
Сервис получает только свой TLS-ключ (Docker secret) и логинится им (`app/openbao.py`). Root token, SecretID, `BAO_TOKEN` в окружении приложения не используются. Каждые 15 с токен продлевается или запрашивается заново, а новая версия KV-секрета передаётся в callback (ротация без рестарта).
## Действия оператора
```sh
sh scripts/bao.sh init # один раз: 3 доли unseal, порог 2, root token -> secrets/operator/ (0600); затем unseal
sh scripts/bao.sh bootstrap # идемпотентно: KV v2 `messenger/`, политики, роли cert auth, секреты сервисов
sh scripts/bao.sh status
sh scripts/bao.sh unseal # ПОСЛЕ КАЖДОГО рестарта OpenBao (он стартует запечатанным)
sh scripts/bao.sh seal # учебный инцидент
sh scripts/bao.sh rotate-worker-db # новый пароль роли app_worker: ALTER ROLE + новая версия messenger/worker
```
**После рестарта OpenBao:**
- уже работающие API и worker продолжают работать на полученных секретах;
- `scripts/bao.sh unseal` возвращает возможность входа;
- сервис, перезапущенный при запечатанном OpenBao, ждёт до `BAO_STARTUP_TIMEOUT_SECONDS` (120 с) и завершается с кодом 78, Docker перезапускает его до unseal;
- restart hook `scripts/restart.sh` делает unseal сам, потому что его запускает оператор.
**Восстановление и компрометация:**
- утрата root token — `bao operator generate-root` с порогом долей;
- компрометация роли сервиса — `bao delete auth/cert/certs/<role>` (новые логины невозможны), отзыв токенов по accessor, ротация секретов в KV, новый сертификат.
**Учебный компромисс.** Все 3 доли и root token лежат у одного оператора на хосте стенда (`secrets/operator/`, вне Git и образов, 0600). В production доли раздаются разным людям, root token отзывается после bootstrap, а для autounseal используется KMS или HSM.
+21
View File
@@ -0,0 +1,21 @@
# OpenBao 2.1.0 server mode (NOT dev mode): persistent file storage on the volume `baodata`, TLS listener.
# The listener asks for a client certificate (tls_client_ca_file) because services authenticate with the cert
# auth method; requests without a certificate are still possible for the operator CLI with a token.
ui = false
disable_mlock = true # container without IPC_LOCK; swap is not configured in the lab VM (documented trade-off)
api_addr = "https://openbao:8200"
# Single node: no cluster listener at all (2.6 opens 8201 by default; nothing should listen without a purpose).
disable_clustering = true
log_level = "info"
storage "file" {
path = "/openbao/file"
}
listener "tcp" {
address = "0.0.0.0:8200"
tls_cert_file = "/run/secrets/openbao.crt"
tls_key_file = "/run/secrets/openbao.key"
tls_client_ca_file = "/run/secrets/ca.crt"
tls_min_version = "tls12"
}
+7
View File
@@ -0,0 +1,7 @@
# API replicas: read ONLY their own KV v2 secret (data + metadata for version checks). Nothing else.
path "messenger/data/api" {
capabilities = ["read"]
}
path "messenger/metadata/api" {
capabilities = ["read"]
}
@@ -0,0 +1,7 @@
# Worker: read ONLY its own KV v2 secret. It cannot read the API secret (other DB role, S3 user, Valkey ACL).
path "messenger/data/worker" {
capabilities = ["read"]
}
path "messenger/metadata/worker" {
capabilities = ["read"]
}
+111
View File
@@ -0,0 +1,111 @@
# OpenTelemetry Collector (contrib 0.120.0): receiver/processor/exporter only, it stores nothing long-term.
# API, worker --OTLP/HTTPS:4318 (mTLS)--> collector --mTLS--> Prometheus (OTLP), Loki (OTLP), Tempo (OTLP/gRPC)
# lab7: the receiver trusts ONLY the "Telemetry Ingest CA" (client_ca_file): api-otel / worker-otel certificates.
# A certificate from the root CA (LB, api, worker identities) or no certificate cannot push telemetry.
# Outbound: the collector presents collector-out (Telemetry Clients CA) and verifies each backend's server cert.
extensions:
health_check:
endpoint: 127.0.0.1:13133 # loopback only (no Docker healthcheck: distroless image)
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 4194304 # 4 MiB per OTLP request
tls:
cert_file: /run/secrets/server.crt
key_file: /run/secrets/server.key
client_ca_file: /run/secrets/ingest-ca.crt
min_version: "1.2"
prometheus/self:
config:
scrape_configs:
- job_name: otel-collector
scrape_interval: 15s
static_configs: [{targets: ["127.0.0.1:8888"]}]
processors:
# First in every pipeline: refuse new data (receivers answer 503 -> SDK retries/drops) before the container OOMs.
# Container limit is 256 MiB; soft limit = limit_mib - spike_limit_mib = 140 MiB.
memory_limiter:
check_interval: 1s
limit_mib: 180
spike_limit_mib: 40
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
# Defense in depth: even if an instrumentation adds them, these attributes never leave the collector.
# url.query would contain ?filename=...; headers could contain Authorization.
attributes/scrub:
actions:
- {key: url.query, action: delete}
- {key: url.full, action: delete}
- {key: http.target, action: delete}
- {key: http.url, action: delete}
- {key: http.request.header.authorization, action: delete}
- {key: http.request.header.cookie, action: delete}
- {key: db.query.parameters, action: delete}
exporters:
# Every exporter verifies the backend against the lab root CA and authenticates as collector-out (mTLS).
otlphttp/prometheus:
endpoint: https://prometheus:9090/api/v1/otlp
tls: &client_tls
ca_file: /run/secrets/ca.crt
cert_file: /run/secrets/client.crt
key_file: /run/secrets/client.key
min_version: "1.2"
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
otlphttp/loki:
endpoint: https://loki:3100/otlp
tls: *client_tls
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
otlp/tempo:
endpoint: tempo:4317
tls: *client_tls
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
service:
extensions: [health_check]
telemetry:
# Stable identity of the collector's own metrics (default is a random UUID per restart -> label churn).
resource:
service.instance.id: otel-collector
logs:
level: info
metrics:
level: detailed
# lab7: self-metrics are served on LOOPBACK only and scraped by the collector itself (receiver
# prometheus/self), then exported through the normal metrics pipeline over mTLS. Nothing is reachable from the
# network in plaintext. (Direct OTLP push of internal telemetry ignored client certificates in 0.120.0.)
readers:
- pull:
exporter:
prometheus:
host: 127.0.0.1
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp, prometheus/self]
processors: [memory_limiter, batch]
exporters: [otlphttp/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlphttp/loki]
+5
View File
@@ -0,0 +1,5 @@
# Default profile: no extra processing. All spans that the SDK head-sampled are exported.
# Selected with COLLECTOR_PROFILE=default (see compose.yaml); merged over collector.yaml.
processors:
batch:
timeout: 2s
+31
View File
@@ -0,0 +1,31 @@
# Track 1 "balanced" profile (COLLECTOR_PROFILE=tail): SDK keeps 100% head sampling, the collector decides per
# whole trace after decision_wait. Keeps: every trace with an error, every trace slower than 500 ms, every trace of
# the attachment pipeline (rare, async, valuable), and 10% of the remaining ordinary traffic.
# Metrics are computed in the SDK BEFORE any sampling, so RPS/error rate/percentiles are unaffected.
processors:
tail_sampling:
decision_wait: 10s
num_traces: 20000
expected_new_traces_per_sec: 200
decision_cache:
sampled_cache_size: 50000
policies:
- name: keep-errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: keep-slow
type: latency
latency: {threshold_ms: 500}
- name: keep-async-pipeline
type: string_attribute
string_attribute: {key: messaging.system, values: [rabbitmq]}
- name: baseline-10pct
type: probabilistic
probabilistic: {sampling_percentage: 10}
service:
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, tail_sampling, batch]
exporters: [otlp/tempo]
+8
View File
@@ -0,0 +1,8 @@
# PostgreSQL client authentication (lab7). Network clients: TLS only (hostssl) + SCRAM-SHA-256; plaintext refused.
# "local" = Unix socket inside the container only (healthcheck, operator `docker compose exec psql`): whoever can
# exec into the container already controls it, so trust adds no new access path.
local all all trust
hostssl all all 0.0.0.0/0 scram-sha-256
hostssl all all ::/0 scram-sha-256
host all all 0.0.0.0/0 reject
host all all ::/0 reject
+5
View File
@@ -0,0 +1,5 @@
# Client side of the container healthcheck (promtool check healthy): verify the server, present the operator cert.
tls_config:
ca_file: /run/secrets/ca.crt
cert_file: /run/secrets/health.crt
key_file: /run/secrets/health.key
+25
View File
@@ -0,0 +1,25 @@
# Prometheus v3.2.1: metrics store. Metrics arrive by OTLP push (--web.enable-otlp-receiver, /api/v1/otlp).
# The web listener is HTTPS with mandatory client certificates (web.yml).
global:
scrape_interval: 15s
evaluation_interval: 15s
otlp:
# job=service.name, instance=service.instance.id are set by Prometheus; these become extra labels.
promote_resource_attributes:
- service.name
- service.instance.id
- service.version
- deployment.environment.name
storage:
tsdb:
# OTLP pushes from several SDKs/batches may arrive slightly out of order.
out_of_order_time_window: 10m
rule_files:
- /etc/prometheus/rules/*.yml
# lab7: no scrape jobs. Every signal arrives by OTLP push over mTLS (application metrics through the Collector,
# Collector self-metrics directly). A scrape of a plaintext metrics port would be an unauthenticated side channel.
scrape_configs: []
+111
View File
@@ -0,0 +1,111 @@
# Recording rules (SLI/SLO) and alert rules. Evaluated by Prometheus every 15 s; state (pending/firing/inactive)
# is visible in Prometheus /alerts, the API /api/v1/alerts and in Grafana (alert list panel). No external delivery.
# Runbooks: docs/runbooks.md (anchors in runbook_url).
#
# Label mapping (OTLP -> Prometheus 3): service.name -> service_name, service.instance.id -> instance,
# http.route -> http_route, http.response.status_code -> http_response_status_code, job.type -> job_type.
# HTTP SLI denominator: every business request of messenger-api (health probes are not instrumented).
# Numerator of errors: 5xx only. 4xx (wrong password, 404 of a foreign chat, 409) are client outcomes, not failures.
groups:
- name: messenger-sli
interval: 15s
rules:
- record: messenger:http_requests:rate2m
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[2m]))
- record: messenger:http_errors_5xx:rate2m
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[2m])) or vector(0)
- record: messenger:http_error_ratio:rate2m
expr: messenger:http_errors_5xx:rate2m / messenger:http_requests:rate2m
- record: messenger:http_latency_p95:rate5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[5m])))
# SLO window: rolling 30 min (lab-sized; production would use 28-30 days with the same formulas).
# Availability SLO: 99% of business requests without 5xx.
- record: messenger:slo_availability:ratio30m
expr: |
1 - (
(sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[30m])) or vector(0))
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
)
# Latency SLO: 95% of business requests complete in <= 500 ms (bucket le="0.5").
- record: messenger:slo_latency:ratio30m
expr: |
sum(increase(http_server_request_duration_seconds_bucket{service_name="messenger-api",le="0.5"}[30m]))
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
- record: messenger:worker_saturation:ratio
# OTLP-pushed series have no staleness markers: a recreated worker would stay in the denominator for the
# 5 min lookback. Only samples fresher than 30 s (3 export intervals) are counted.
expr: |
sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 30))
/ sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))
- name: messenger-alerts
rules:
- alert: MessengerApiHighErrorRate
expr: messenger:http_error_ratio:rate2m > 0.05 and messenger:http_requests:rate2m > 0.2
for: 1m
labels:
severity: page
annotations:
summary: "API: доля 5xx {{ $value | humanizePercentage }} за 2 мин (порог 5%)"
description: "Больше 5% бизнес-запросов messenger-api завершаются 5xx (окно 2m, держится 1m). Расходуется бюджет SLO доступности 99%."
runbook_url: "docs/runbooks.md#messengerapihigherrorrate"
- alert: MessengerQueueLagGrowing
expr: max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60
for: 1m
labels:
severity: page
annotations:
summary: "Очередь вложений: старейшее задание ждёт {{ $value | humanizeDuration }}"
description: "Принятое задание не завершено дольше 60 с (порог), условие держится 1m: worker остановлен, перегружен или не может писать в S3/DB."
runbook_url: "docs/runbooks.md#messengerqueuelaggrowing"
- alert: MessengerApiLatencyHigh
expr: messenger:http_latency_p95:rate5m > 1
for: 2m
labels:
severity: ticket
annotations:
summary: "API p95 = {{ $value | humanizeDuration }} (порог 1 s, окно 5m)"
description: "p95 из histogram buckets всех реплик выше 1 s."
runbook_url: "docs/runbooks.md#messengerapilatencyhigh"
- alert: MessengerDbPoolWaiting
expr: max(db_client_connection_pending_requests) > 0
for: 1m
labels:
severity: ticket
annotations:
summary: "Запросы ждут соединение из пула БД ({{ $value }} в очереди)"
description: "Пул psycopg исчерпан дольше минуты: БД медленная/блокирована или пул мал."
runbook_url: "docs/runbooks.md#messengerdbpoolwaiting"
- alert: MessengerWorkerSaturated
expr: messenger:worker_saturation:ratio > 0.9
for: 5m
labels:
severity: ticket
annotations:
summary: "Worker занят на {{ $value | humanizePercentage }} ёмкости 5 минут"
runbook_url: "docs/runbooks.md#messengerworkersaturated"
- alert: TelemetryExportFailing
# Each term has "or vector(0)": the *_failed_* series do not exist until the first failure, and a sum with an
# empty operand is empty (the alert would never fire). Early signal: exporter queue filling up (backend
# down, retries in progress); late signal: data dropped (queue full) or retries exhausted.
expr: |
(
(sum(rate(otelcol_exporter_send_failed_spans_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_send_failed_log_records_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_enqueue_failed_spans_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_enqueue_failed_log_records_total[2m])) or vector(0))
) > 0
or
max(max by (exporter) (otelcol_exporter_queue_size) / max by (exporter) (otelcol_exporter_queue_capacity)) > 0.2
for: 1m
labels:
severity: ticket
annotations:
summary: "Collector не может доставить telemetry в backend"
description: "Очередь exporter заполнена >20% или данные отбрасываются: Tempo/Loki/Prometheus недоступен. Бизнес-API не затронут, RAM collector ограничена очередью и memory_limiter."
runbook_url: "docs/runbooks.md#telemetryexportfailing"
+9
View File
@@ -0,0 +1,9 @@
# Prometheus web listener (lab7): TLS + mandatory client certificate from the "Telemetry Clients CA" only
# (collector-out for OTLP ingest, grafana-out for queries, operator for evidence scripts and the healthcheck).
# Root-signed service identities (api, worker, lb) are not in this CA and are refused at the handshake.
tls_server_config:
cert_file: /run/secrets/server.crt
key_file: /run/secrets/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/secrets/telemetry-ca.crt
min_version: TLS12
+25
View File
@@ -0,0 +1,25 @@
{
"vhosts": [{"name": "messenger"}],
"users": [
{"name": "api.client.messenger", "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []},
{"name": "worker.client.messenger", "password_hash": "", "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}
],
"permissions": [
{"user": "api.client.messenger", "vhost": "messenger", "configure": "^$", "write": "^attachments$", "read": "^$"},
{"user": "worker.client.messenger", "vhost": "messenger", "configure": "^$", "write": "^$", "read": "^attachments\\.process$"}
],
"exchanges": [
{"name": "attachments", "vhost": "messenger", "type": "direct", "durable": true, "auto_delete": false, "internal": false, "arguments": {}},
{"name": "attachments.dlx", "vhost": "messenger", "type": "fanout", "durable": true, "auto_delete": false, "internal": false, "arguments": {}}
],
"queues": [
{"name": "attachments.process", "vhost": "messenger", "durable": true, "auto_delete": false,
"arguments": {"x-queue-type": "quorum", "x-dead-letter-exchange": "attachments.dlx", "x-delivery-limit": 20}},
{"name": "attachments.dlq", "vhost": "messenger", "durable": true, "auto_delete": false,
"arguments": {"x-queue-type": "quorum"}}
],
"bindings": [
{"source": "attachments", "vhost": "messenger", "destination": "attachments.process", "destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}},
{"source": "attachments.dlx", "vhost": "messenger", "destination": "attachments.dlq", "destination_type": "queue", "routing_key": "", "arguments": {}}
]
}
+1
View File
@@ -0,0 +1 @@
[rabbitmq_auth_mechanism_ssl].
+25
View File
@@ -0,0 +1,25 @@
# RabbitMQ 4.0 (lab7): TLS-only AMQP with mandatory client certificates. Identity = certificate CN, mapped to a
# passwordless user from definitions.json (auth mechanism EXTERNAL). No password users, no guest, no management UI.
listeners.tcp = none
listeners.ssl.default = 5671
ssl_options.cacertfile = /run/secrets/ca.crt
ssl_options.certfile = /run/secrets/server.crt
ssl_options.keyfile = /run/secrets/server.key
ssl_options.verify = verify_peer
ssl_options.fail_if_no_peer_cert = true
ssl_options.versions.1 = tlsv1.3
ssl_options.versions.2 = tlsv1.2
auth_mechanisms.1 = EXTERNAL
ssl_cert_login_from = common_name
# Users, permissions, vhost and topology come from the definitions file on every boot.
definitions.import_backend = local_filesystem
definitions.local.path = /etc/rabbitmq/definitions.json
definitions.skip_if_unchanged = true
loopback_users = none
consumer_timeout = 600000
disk_free_limit.absolute = 200MB
vm_memory_high_watermark.relative = 0.6
log.console.level = warning
# Erlang distribution (CLI tools, clustering) is loopback-only: no cluster here, and rabbitmqctl runs inside the
# container. Otherwise ports 25672/4369 would be a cookie-protected plaintext channel reachable from the network.
distribution.listener.interface = 127.0.0.1
+40
View File
@@ -0,0 +1,40 @@
# Tempo 2.7.1 single binary: trace store (local blocks on a volume). Receives OTLP/gRPC (mTLS) from the Collector.
stream_over_http_enabled: true
server:
http_listen_port: 3200
grpc_listen_address: 127.0.0.1 # internal gRPC of the single binary: loopback only
log_level: warn
# lab7: query API over HTTPS with mandatory client certificates from the "Telemetry Clients CA".
http_tls_config:
cert_file: /run/secrets/server.crt
key_file: /run/secrets/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/secrets/telemetry-ca.crt
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
# OTLP ingest from the Collector only: mTLS with the same client CA (collector-out).
tls:
cert_file: /run/secrets/server.crt
key_file: /run/secrets/server.key
client_ca_file: /run/secrets/telemetry-ca.crt
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
query_frontend:
search:
max_result_limit: 10000
usage_report:
reporting_enabled: false
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
# Valkey (lab7): TLS-only (plaintext port 0), ACL users rendered at start into the tmpfs from a Docker secret.
# default disabled
# api session cache + login rate limit keys only, a minimal command set (no FLUSH*, CONFIG, KEYS, SCAN, EVAL)
# health PING only (container healthcheck)
# Cache only: no persistence; PostgreSQL stays the source of truth for sessions.
set -eu
umask 077
{
echo "user default off"
echo "user api on >$(cat /run/secrets/valkey_api_password) resetkeys ~session:* ~ratelimit:* resetchannels -@all +get +set +del +incr +incrby +expire +ttl +ping +hello +client|setinfo +client|setname +select"
echo "user health on >health resetkeys -@all +ping"
} > /tmp/users.acl
exec valkey-server \
--port 0 \
--tls-port 6379 \
--tls-cert-file /run/secrets/server.crt \
--tls-key-file /run/secrets/server.key \
--tls-ca-cert-file /run/secrets/ca.crt \
--tls-auth-clients no \
--aclfile /tmp/users.acl \
--save "" --appendonly no \
--maxmemory 64mb --maxmemory-policy allkeys-lru \
--dir /data
+46
View File
@@ -0,0 +1,46 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name bytea NOT NULL
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
created_at timestamptz NOT NULL
);
CREATE INDEX chats_created_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
text bytea NOT NULL
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
created_at timestamptz NOT NULL
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
key text NOT NULL
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
request_body_hash text NOT NULL,
response_json text NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+28
View File
@@ -0,0 +1,28 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
filename text NOT NULL
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL,
status text NOT NULL DEFAULT 'ready'
CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_size;
ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000);
@@ -0,0 +1,5 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed'));
ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT;
CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key);
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
+75
View File
@@ -0,0 +1,75 @@
-- lab5: upload -> staging -> job/outbox -> broker -> worker -> final prefix + variants.
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed'));
-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker.
ALTER TABLE attachments ADD COLUMN staging_key text;
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL;
-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0),
width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0),
height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0),
sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL,
PRIMARY KEY (attachment_id, pipeline_version, name)
);
-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry.
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
state text NOT NULL
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')),
attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1),
max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1),
next_attempt_at timestamptz NOT NULL,
lease_owner text,
lease_until timestamptz,
error_code text,
last_error text,
created_at timestamptz NOT NULL,
first_started_at timestamptz,
started_at timestamptz,
finished_at timestamptz,
updated_at timestamptz NOT NULL,
UNIQUE (attachment_id, pipeline_version)
);
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing';
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed';
CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing');
-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms.
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
payload jsonb NOT NULL,
available_at timestamptz NOT NULL,
published_at timestamptz,
publish_attempts integer NOT NULL DEFAULT 0,
last_error text,
created_at timestamptz NOT NULL
);
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL;
CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL;
-- Operator-visible history of every state change of a job (including explicit replay).
CREATE TABLE job_audit (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL,
action text NOT NULL,
actor text NOT NULL,
attempt integer NOT NULL,
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz NOT NULL
);
CREATE INDEX job_audit_job_idx ON job_audit (job_id, id);
+29
View File
@@ -0,0 +1,29 @@
-- lab7: separate least-privilege DB roles for the services. Passwords are NOT here: scripts/migrate.py sets them
-- from operator bootstrap files and the services read them from OpenBao. The owner role `messenger` is only used
-- by migrations and the operator.
DO $$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'app_api') THEN
CREATE ROLE app_api LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT CONNECTION LIMIT 40;
END IF;
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'app_worker') THEN
CREATE ROLE app_worker LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT CONNECTION LIMIT 20;
END IF;
END
$$;
REVOKE ALL ON DATABASE messenger FROM PUBLIC;
GRANT CONNECT ON DATABASE messenger TO app_api, app_worker;
REVOKE ALL ON SCHEMA public FROM PUBLIC;
GRANT USAGE ON SCHEMA public TO app_api, app_worker;
-- API: business tables of every endpoint, the outbox relay, the reconciler and scripts/jobs.py.
GRANT SELECT, INSERT, UPDATE, DELETE ON users, chats, chat_members, messages, message_attachments, sessions,
idempotency_keys, attachments, attachment_variants, attachment_jobs, outbox, job_audit TO app_api;
-- Worker: only the processing pipeline. No users/sessions/messages/chats, no DELETE anywhere.
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO app_worker;
GRANT SELECT, INSERT ON attachment_variants, job_audit TO app_worker;
GRANT INSERT ON outbox TO app_worker; -- retry scheduling writes a new outbox row; publishing stays in the API
GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO app_api, app_worker;
+60
View File
@@ -0,0 +1,60 @@
aio-pika==9.5.5
aiormq==6.8.1
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
asgiref==3.13.0
boto3==1.35.81
botocore==1.35.81
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
Deprecated==3.0.1
exceptiongroup==1.2.2
fastapi==0.115.6
googleapis-common-protos==1.75.0
h11==0.16.0
httptools==0.9.0
idna==3.20
importlib_metadata==8.5.0
jmespath==1.0.1
multidict==6.1.0
opentelemetry-api==1.30.0
opentelemetry-exporter-otlp-proto-common==1.30.0
opentelemetry-exporter-otlp-proto-http==1.30.0
opentelemetry-instrumentation==0.51b0
opentelemetry-instrumentation-asgi==0.51b0
opentelemetry-instrumentation-botocore==0.51b0
opentelemetry-instrumentation-fastapi==0.51b0
opentelemetry-propagator-aws-xray==1.0.2
opentelemetry-proto==1.30.0
opentelemetry-sdk==1.30.0
opentelemetry-semantic-conventions==0.51b0
opentelemetry-util-http==0.51b0
packaging==26.3
pamqp==3.3.0
pillow==13.0.0
propcache==0.2.1
protobuf==5.29.6
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
pycparser==3.11
pydantic==2.14.1
pydantic_core==2.50.1
python-dateutil==2.9.0.post0
requests==2.34.2
s3transfer==0.10.4
six==1.17.0
starlette==0.41.3
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.0
uvloop==0.23.0
valkey==6.2.0
wrapt==1.17.3
yarl==1.18.3
zipp==4.1.2
Executable
+126
View File
@@ -0,0 +1,126 @@
#!/bin/sh
# OpenBao operator tool. Unseal shares and the root token live ONLY in secrets/operator/ (git-ignored, 0600) and
# are never printed. Values are passed to the CLI through the environment of `docker compose exec`, not argv.
#
# scripts/bao.sh init initialise once: 3 unseal shares, threshold 2; then unseal
# scripts/bao.sh unseal unseal after every OpenBao restart (operator action; the API does not do it)
# scripts/bao.sh bootstrap KV v2 mount, policies, cert auth roles, service secrets (idempotent)
# scripts/bao.sh status sealed/initialised state
# scripts/bao.sh rotate-worker-db new password for DB role app_worker: PostgreSQL + OpenBao (level 4)
# scripts/bao.sh seal seal OpenBao (incident drill: services keep cached secrets, new ones must wait)
#
# Production note: each share would go to a different person; here one operator holds them on one host.
set -eu
cd "$(dirname "$0")/.."
OP=secrets/operator
BS=secrets/bootstrap
ENVS="-e BAO_ADDR=https://127.0.0.1:8200 -e BAO_CACERT=/run/secrets/ca.crt"
bao() { docker compose exec -T $ENVS openbao bao "$@"; }
bao_root() { BAO_TOKEN="$(cat "$OP/root-token")" docker compose exec -T $ENVS -e BAO_TOKEN openbao bao "$@"; }
wait_up() {
i=0
until docker compose exec -T $ENVS openbao sh -c 'bao status >/dev/null 2>&1; [ $? -ne 1 ]'; do
i=$((i + 1)); [ $i -gt 60 ] && { echo "openbao not reachable" >&2; exit 1; }; sleep 1
done
}
sealed() { ! docker compose exec -T $ENVS openbao bao status >/dev/null 2>&1; }
unseal() {
wait_up
if ! sealed; then echo "openbao: already unsealed"; return 0; fi
for n in 1 2; do
UNSEAL_KEY="$(cat "$OP/unseal-$n")" docker compose exec -T $ENVS -e UNSEAL_KEY openbao \
sh -c 'bao operator unseal "$UNSEAL_KEY" >/dev/null'
done
sealed && { echo "openbao: still sealed" >&2; exit 1; }
echo "openbao: unsealed with 2 of 3 operator shares"
}
init() {
wait_up
if bao status -format=json 2>/dev/null | python3 -c 'import json,sys;sys.exit(0 if json.load(sys.stdin)["initialized"] else 1)'; then
echo "openbao: already initialised"; unseal; return 0
fi
umask 077; mkdir -p "$OP"
bao operator init -key-shares=3 -key-threshold=2 -format=json | python3 -c '
import json, sys
d = json.load(sys.stdin)
for i, key in enumerate(d["unseal_keys_b64"], 1):
open("'"$OP"'/unseal-%d" % i, "w").write(key)
open("'"$OP"'/root-token", "w").write(d["root_token"])
'
chmod 600 "$OP"/*
echo "openbao: initialised; 3 shares and root token stored in $OP/ (0600, not printed)"
unseal
}
bootstrap() {
unseal
bao_root secrets list -format=json | grep -q '"messenger/"' || bao_root secrets enable -path=messenger -version=2 kv
bao_root auth list -format=json | grep -q '"cert/"' || bao_root auth enable cert
for p in messenger-api messenger-worker; do
docker compose exec -T openbao sh -c "cat > /tmp/$p.hcl" < "infra/openbao/policies/$p.hcl"
bao_root policy write "$p" "/tmp/$p.hcl" >/dev/null
done
docker compose exec -T openbao sh -c 'cp /run/secrets/ca.crt /tmp/ca.crt'
# Identity = client certificate chained to the lab root CA AND an exact common name. Short-lived tokens.
bao_root write auth/cert/certs/api certificate=@/tmp/ca.crt allowed_common_names=api.client.messenger \
token_policies=messenger-api token_ttl=15m token_max_ttl=1h >/dev/null
bao_root write auth/cert/certs/worker certificate=@/tmp/ca.crt allowed_common_names=worker.client.messenger \
token_policies=messenger-worker token_ttl=15m token_max_ttl=1h >/dev/null
kv_api
kv_worker
echo "openbao: mount messenger/ (kv v2), policies messenger-api/worker, cert roles api/worker, secrets written"
}
# KV values are read from secrets/bootstrap and passed through stdin as JSON (never on a command line).
kv_api() {
python3 - "$BS" <<'PY' | bao_root kv put -mount=messenger api - >/dev/null
import json, sys
b = sys.argv[1]
r = lambda n: open("%s/%s" % (b, n)).read().strip()
print(json.dumps({
"database_url": "postgresql://app_api:%s@postgres:5432/messenger" % r("db_api_password"),
"s3_access_key_id": r("s3_api_access_key"), "s3_secret_access_key": r("s3_api_secret_key"),
"cache_url": "rediss://api:%s@valkey:6379/0" % r("valkey_api_password")}))
PY
}
kv_worker() {
python3 - "$BS" <<'PY' | bao_root kv put -mount=messenger worker - >/dev/null
import json, sys
b = sys.argv[1]
r = lambda n: open("%s/%s" % (b, n)).read().strip()
print(json.dumps({
"database_url": "postgresql://app_worker:%s@postgres:5432/messenger" % r("db_worker_password"),
"s3_access_key_id": r("s3_worker_access_key"), "s3_secret_access_key": r("s3_worker_secret_key")}))
PY
}
rotate_worker_db() {
unseal
umask 077
python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > "$BS/db_worker_password.new"
chmod 644 "$BS/db_worker_password.new"
# 1) PostgreSQL first (open connections keep working), 2) OpenBao: the worker picks up the new version.
NEWPW="$(cat "$BS/db_worker_password.new")" docker compose exec -T -e NEWPW postgres sh -c \
'psql -v ON_ERROR_STOP=1 -q -U messenger -d messenger -v pw="$NEWPW" <<SQL
ALTER ROLE app_worker PASSWORD :'"'"'pw'"'"';
SQL'
mv "$BS/db_worker_password.new" "$BS/db_worker_password"
kv_worker
echo "rotated: app_worker password changed in PostgreSQL and OpenBao (messenger/worker, new KV version)"
}
case "${1:-status}" in
init) init ;;
unseal) unseal ;;
bootstrap) bootstrap ;;
rotate-worker-db) rotate_worker_db ;;
seal) wait_up; bao_root operator seal >/dev/null; echo "openbao: sealed" ;;
status) wait_up; bao status | grep -E "Initialized|Sealed|Version|Storage" ;;
*) echo "usage: $0 init|unseal|bootstrap|status|rotate-worker-db|seal" >&2; exit 2 ;;
esac
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'service_name="messenger-api"'
HTTP = "http_server_request_duration_seconds"
_ids = iter(range(1, 1000))
def target(expr, legend="", ref="A", instant=False):
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
"instant": instant}
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
panel = {
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"stacking": {"mode": "normal" if stack else "none"},
"showPoints": "never"}},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
}
if extra:
panel["fieldConfig"]["defaults"].update(extra)
return panel
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
steps = thresholds or [{"color": "green", "value": None}]
panel = {
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
"overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
"targets": [target(expr, title)],
}
if decimals is not None:
panel["fieldConfig"]["defaults"]["decimals"] = decimals
return panel
def row(title, y):
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
"panels": []}
def q(p, expr_inner):
return "histogram_quantile(%s, %s)" % (p, expr_inner)
panels = []
y = 0
# ------------------------------------------------------------------ SLO and alerts
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
{"color": "green", "value": warn}]
panels += [
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
thresholds=red_yellow(0.99, 0.995), decimals=2),
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
"viewMode": "list", "datasource": "Prometheus"}},
]
y += 4
panels += [
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
]
y += 4
# ------------------------------------------------------------------ API
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
panels += [
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
ts("RPS по классу статуса", [(
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
"{{status_class}}")], "reqps", 12, y, stack=True),
]
y += 8
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
panels += [
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
ts("Доля ошибок 5xx", [
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
"5xx / все", "A")], "percentunit", 12, y,
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
]
y += 8
panels += [
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
"{{http_route}}")], "s", 0, y),
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
"short", 12, y),
]
y += 8
# ------------------------------------------------------------------ worker and queue
panels.append(row("Очередь и worker", y)); y += 1
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
panels += [
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"{{outcome}}")], "ops", 0, y, stack=True,
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
(q(0.99, jb), "p99", "C")], "s", 12, y),
]
y += 8
panels += [
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
"thresholdsStyle": {"mode": "line"}},
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
{"color": "red", "value": 60}]}}),
]
y += 8
panels += [
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
ts("Насыщение worker: занято / ёмкость", [
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ resources
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
panels += [
ts("Пул БД: used / idle / waiting", [
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
"{{service_name}} {{db_client_connection_state}}", "A"),
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
ts("CPU процессов (ядра)", [
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
"{{service_name}} {{instance}}", "A"),
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
desc="worker: self + завершённые дочерние процессы обработки изображений."),
]
y += 8
panels += [
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
"sent {{exporter}}", "B"),
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ logs and traces
panels.append(row("Логи и трассы", y)); y += 1
panels += [
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
"targets": [{"datasource": LOKI, "refId": "A",
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
]
y += 10
dashboard = {
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
"templating": {"list": []},
}
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
"bytes": "байты"}
DOC = OUT.parents[3] / "docs/dashboard.md"
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
for panel in panels:
if panel["type"] == "row":
lines += ["## " + panel["title"], ""]
continue
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
if panel.get("description"):
lines.append("— " + panel["description"])
lines.append("")
if panel["type"] == "alertlist":
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
for t in panel.get("targets", []):
query = t.get("expr") or t.get("query")
legend = t.get("legendFormat", "")
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
lines.append("")
DOC.write_text("\n".join(lines), encoding="utf-8")
print("wrote docs/dashboard.md")
+34
View File
@@ -0,0 +1,34 @@
#!/bin/sh
# Operator step 1 (after scripts/pki.sh init): local .env and random bootstrap credentials in secrets/bootstrap/.
# Nothing is printed. Existing files are kept, so a re-run never changes the passwords of a live stand.
#
# Who gets what:
# stores (postgres, minio, valkey, grafana) read their OWN admin/ACL material from Docker secrets;
# API and worker get NO password files at all: scripts/bao.sh bootstrap copies their credentials into OpenBao
# and the services read them there after authenticating with their client certificate.
set -eu
cd "$(dirname "$0")/.."
[ -f .env ] || cp .env.example .env
umask 077
mkdir -p secrets/bootstrap
cd secrets/bootstrap
put() { [ -f "$1" ] || printf '%s' "$2" > "$1"; }
random() { python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')"; }
put postgres_password "$(random)" # superuser `messenger`: migrations and operator only
put db_api_password "$(random)" # role app_api
put db_worker_password "$(random)" # role app_worker
put minio_root_user messenger_root
put minio_root_password "$(random)"
put s3_api_access_key messenger-api
put s3_api_secret_key "$(random)"
put s3_worker_access_key messenger-worker
put s3_worker_secret_key "$(random)"
put valkey_api_password "$(random)" # ACL user `api`
put grafana_admin_password "$(random)"
put migrate_database_url "postgresql://messenger:$(cat postgres_password)@postgres:5432/messenger"
chmod 644 ./* # read by container users through Docker secrets; the directory itself is 700
cd ..
chmod 700 . bootstrap
echo "bootstrap credentials ready in secrets/bootstrap/ (.env present)"
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...).
failed registry of failed jobs (the "DLQ" of business failures)
show <attachment_id> job state + audit trail
replay <attachment_id> --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing
stats counts by state and outbox backlog
Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md).
"""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import jobs # noqa: E402
from app.cursor import fmt # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
def out(value) -> None:
print(json.dumps(value, ensure_ascii=False, indent=2, default=str))
async def main() -> int:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("failed")
sub.add_parser("stats")
show = sub.add_parser("show")
show.add_argument("attachment_id")
rp = sub.add_parser("replay")
rp.add_argument("attachment_id")
rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)")
args = parser.parse_args()
db = Database()
await db.open()
try:
if args.cmd == "failed":
async with db.connection() as conn:
out(await jobs.failed_jobs(conn))
elif args.cmd == "stats":
async with db.connection() as conn:
cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state")
states = {r["state"]: r["n"] for r in await cur.fetchall()}
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
backlog = (await cur.fetchone())["n"]
out({"jobs": states, "outbox_unpublished": backlog})
elif args.cmd == "show":
async with db.connection() as conn:
cur = await conn.execute(
"SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until,"
" a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.attachment_id = %s", (args.attachment_id,))
job = await cur.fetchone()
cur = await conn.execute(
"SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
(args.attachment_id,))
trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()]
cur = await conn.execute(
"SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts"
" FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id",
(args.attachment_id,))
outbox = await cur.fetchall()
out({"job": job, "audit": trail, "outbox": outbox})
elif args.cmd == "replay":
s3 = S3Storage()
await run_in_threadpool(s3.open)
async def exists(key):
return await run_in_threadpool(s3.exists, key)
result = await db.in_transaction(
lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists))
out(result)
return 0 if result["result"] == "queued" else 3
finally:
await db.close()
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
import hashlib
import os
import pathlib
import sys
import psycopg
ROOT = pathlib.Path(__file__).resolve().parents[1]
MIGRATIONS = ROOT / "migrations"
LOCK_ID = 2026_0004
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE")
if path:
return pathlib.Path(path).read_text(encoding="utf-8").strip()
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
return url
def set_service_passwords(conn) -> None:
"""lab7: passwords of the service roles come from operator bootstrap files (never from Git or the image).
Re-running keeps them in sync with secrets/bootstrap, which scripts/bao.sh also copies into OpenBao."""
from psycopg import sql
for role, env in (("app_api", "DB_API_PASSWORD_FILE"), ("app_worker", "DB_WORKER_PASSWORD_FILE")):
path = os.environ.get(env)
if not path:
continue
with open(path, encoding="utf-8") as handle:
password = handle.read().strip()
conn.execute(sql.SQL("ALTER ROLE {} PASSWORD {}").format(sql.Identifier(role), sql.Literal(password)))
print("password set for role %s" % role)
def main() -> None:
files = sorted(MIGRATIONS.glob("*.sql"))
if not files:
sys.exit("no migration files found in %s" % MIGRATIONS)
tls = {"sslmode": os.environ.get("DATABASE_SSLMODE", "verify-full"),
"sslrootcert": os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt")}
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10, **tls) as conn:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY,"
" checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
applied = {row[0]: row[1] for row in rows}
for path in files:
version = path.stem
body = path.read_text(encoding="utf-8")
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
if version in applied:
if applied[version] != checksum:
sys.exit("migration %s changed after it was applied" % version)
print("skip %s" % version)
continue
with conn.transaction():
conn.execute(body)
conn.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
(version, checksum),
)
print("apply %s" % version)
set_service_passwords(conn)
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
print("migrations complete: %d file(s)" % len(files))
if __name__ == "__main__":
main()
Executable
+190
View File
@@ -0,0 +1,190 @@
#!/bin/sh
# Local lab PKI (operator tool). Private keys stay in secrets/pki/ (git-ignored, never baked into images).
#
# scripts/pki.sh init create CAs and every leaf that does not exist yet (idempotent)
# scripts/pki.sh issue <leaf> (re)issue one leaf with a new key and serial (rotation)
# scripts/pki.sh revoke <leaf> add the CURRENT serial of <leaf> to secrets/pki/revoked-serials.txt
# scripts/pki.sh revoke-serial <hex> revoke a serial that is no longer the current one (after a rotation)
# scripts/pki.sh expired <leaf> test-only: an ALREADY EXPIRED copy of <leaf> (same identity) -> <leaf>-expired.*
# scripts/pki.sh serial <leaf> current serial of <leaf>
# scripts/pki.sh list leaf names, identities, serials and expiry (no key material)
#
# Trust layout (identity is checked by every server, not only the chain):
# root CA "Messenger Lab Root CA" signs every server certificate and the service client identities
# (lb, api, worker) used for LB->API, RabbitMQ (EXTERNAL = certificate CN) and OpenBao cert auth.
# ingest CA (intermediate) signs ONLY api-otel / worker-otel: the Collector trusts nothing else, so a
# root-signed certificate (e.g. the LB one) cannot push telemetry.
# telemetry CA (intermediate) signs ONLY collector-out / grafana-out / operator: Prometheus, Loki and Tempo
# accept only these clients.
# intruder root-signed, valid, but with an identity that no server allows (negative tests).
# foreign CA unrelated CA; foreign-client claims the lb/api/worker names (negative tests of chain checks).
# OpenSSL runs in a pinned container, so the host OpenSSL/LibreSSL version does not matter.
set -eu
cd "$(dirname "$0")/.."
DIR=secrets/pki
IMAGE="alpine/openssl:3.3.2@sha256:5008e829163320a6e8166883c03e68189e8925ade68cde36584dc2a41cfa5248"
LEAF_DAYS="${LEAF_DAYS:-90}"
CA_DAYS=825
# name | signer | usage | CN | DNS/IP SANs
LEAVES='
lb-server root server localhost localhost,nginx,IP:127.0.0.1
lb-client root client lb.client.messenger lb.client.messenger
api-1 root server api-1 api-1,api.internal
api-2 root server api-2 api-2,api.internal
api root client api.client.messenger api.client.messenger
worker root client worker.client.messenger worker.client.messenger
postgres root server postgres postgres
minio root server minio minio
rabbitmq root server rabbitmq rabbitmq
valkey root server valkey valkey
openbao root server openbao openbao,localhost,IP:127.0.0.1
otel-collector root server otel-collector otel-collector
prometheus root server prometheus prometheus,localhost,IP:127.0.0.1
loki root server loki loki,localhost,IP:127.0.0.1
tempo root server tempo tempo,localhost,IP:127.0.0.1
grafana root server grafana grafana,localhost,IP:127.0.0.1
api-otel ingest client api.otel.messenger api.otel.messenger
worker-otel ingest client worker.otel.messenger worker.otel.messenger
collector-out telemetry client collector.out.messenger collector.out.messenger
grafana-out telemetry client grafana.out.messenger grafana.out.messenger
operator telemetry client operator.messenger operator.messenger
intruder root client intruder.client.messenger intruder.client.messenger
'
ossl() { # quiet on success, the OpenSSL error on failure
out=$(docker run --rm -u "$(id -u):$(id -g)" -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" "$@" 2>&1) || {
echo "openssl $1 failed: $out" >&2; exit 1; }
}
ca_init() { # ca_init <name> <subject> [parent]
[ -f "$DIR/$1.crt" ] && return 0
cat > "$DIR/.ext-$1" <<EOF
basicConstraints=critical,CA:true,pathlen:0
keyUsage=critical,keyCertSign,cRLSign
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid
EOF
ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1.key"
if [ $# -eq 2 ]; then
ossl req -x509 -new -key "$1.key" -subj "$2" -days "$CA_DAYS" -sha256 \
-addext "basicConstraints=critical,CA:true,pathlen:1" \
-addext "keyUsage=critical,keyCertSign,cRLSign" -out "$1.crt"
else
ossl req -new -key "$1.key" -subj "$2" -out "$1.csr"
ossl x509 -req -in "$1.csr" -CA "$3.crt" -CAkey "$3.key" -CAcreateserial -days "$CA_DAYS" -sha256 \
-extfile ".ext-$1" -out "$1.crt"
rm -f "$DIR/$1.csr"
fi
rm -f "$DIR/.ext-$1"
echo "created CA $1"
}
leaf_line() { echo "$LEAVES" | awk -v n="$1" '$1==n'; }
issue() { # issue <leaf>
line=$(leaf_line "$1")
[ -n "$line" ] || { echo "unknown leaf: $1" >&2; exit 2; }
set -- $line
name=$1 signer=$2 usage=$3 cn=$4 sans=$5
eku=serverAuth; [ "$usage" = client ] && eku=clientAuth
alt=$(echo "$sans" | tr ',' '\n' | awk '/^IP:/ {print; next} {print "DNS:" $0}' | paste -sd, -)
cat > "$DIR/.ext-$name" <<EOF
basicConstraints=critical,CA:false
keyUsage=critical,digitalSignature,keyEncipherment
extendedKeyUsage=$eku
subjectAltName=$alt
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid
EOF
ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$name.key.new"
ossl req -new -key "$name.key.new" -subj "/O=Messenger Lab/CN=$cn" -out "$name.csr"
ossl x509 -req -in "$name.csr" -CA "$signer.crt" -CAkey "$signer.key" -CAcreateserial -days "$LEAF_DAYS" \
-sha256 -extfile ".ext-$name" -out "$name.crt.new"
# Leaf files carry the chain up to (not including) the root, so intermediates verify against ca.crt alone.
if [ -f "$DIR/$name.crt" ]; then cp "$DIR/$name.crt" "$DIR/$name.crt.previous"; cp "$DIR/$name.key" "$DIR/$name.key.previous"; fi
# Overwrite in place (cat >): the files are bind-mounted Docker secrets, a new inode would not be visible.
if [ "$signer" = root ]; then cat "$DIR/$name.crt.new" > "$DIR/$name.crt"; else cat "$DIR/$name.crt.new" "$DIR/$signer.crt" > "$DIR/$name.crt"; fi
cat "$DIR/$name.key.new" > "$DIR/$name.key"; rm -f "$DIR/$name.key.new"
rm -f "$DIR/$name.crt.new" "$DIR/$name.csr" "$DIR/.ext-$name"
echo "issued $name ($cn) serial=$(serial "$name") valid ${LEAF_DAYS}d"
}
foreign() { # an unrelated CA and a leaf that CLAIMS the LB/api/worker names: negative tests of chain checks
[ -f "$DIR/foreign-client.crt" ] && return 0
ca_init foreign "/O=Somebody Else/CN=Foreign CA"
printf '%s\n' "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature,keyEncipherment" \
"extendedKeyUsage=clientAuth,serverAuth" \
"subjectAltName=DNS:lb.client.messenger,DNS:api.client.messenger,DNS:worker.client.messenger" \
> "$DIR/.ext-foreign-client"
ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out foreign-client.key
ossl req -new -key foreign-client.key -subj "/O=Somebody Else/CN=lb.client.messenger" -out foreign-client.csr
ossl x509 -req -in foreign-client.csr -CA foreign.crt -CAkey foreign.key -CAcreateserial -days 30 -sha256 \
-extfile .ext-foreign-client -out foreign-client.crt
rm -f "$DIR/foreign-client.csr" "$DIR/.ext-foreign-client"
echo "created foreign CA + foreign-client (claims lb/api/worker identities, trusted nowhere)"
}
expired() { # same CN/SAN/EKU as <leaf>, validity in the past (openssl ca allows explicit dates)
line=$(leaf_line "$1"); [ -n "$line" ] || { echo "unknown leaf: $1" >&2; exit 2; }
set -- $line
name=$1 signer=$2 usage=$3 cn=$4 sans=$5
eku=serverAuth; [ "$usage" = client ] && eku=clientAuth
alt=$(echo "$sans" | tr ',' '\n' | awk '/^IP:/ {print; next} {print "DNS:" $0}' | paste -sd, -)
t="$DIR/.ca-$name"; rm -rf "$t"; mkdir -p "$t"; touch "$t/index.txt"; echo 01 > "$t/serial"
printf '%s\n' "[ca]" "default_ca=x" "[x]" "database=/pki/.ca-$name/index.txt" "serial=/pki/.ca-$name/serial" \
"new_certs_dir=/pki/.ca-$name" "default_md=sha256" "policy=p" "unique_subject=no" "[p]" "commonName=supplied" \
"[ext]" "basicConstraints=critical,CA:false" "keyUsage=critical,digitalSignature" "extendedKeyUsage=$eku" \
"subjectAltName=$alt" > "$t/ca.cnf"
ossl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$name-expired.key"
ossl req -new -key "$name-expired.key" -subj "/O=Messenger Lab/CN=$cn" -out ".ca-$name/req.csr"
ossl ca -batch -config ".ca-$name/ca.cnf" -extensions ext -cert "$signer.crt" -keyfile "$signer.key" \
-startdate 20250101000000Z -enddate 20250102000000Z -in ".ca-$name/req.csr" -out "$name-expired.crt" -notext
rm -rf "$t"
echo "issued $name-expired ($cn), validity 2025-01-01..2025-01-02 (already expired, test only)"
}
serial() { docker run --rm -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" x509 -in "$1.crt" -noout -serial | cut -d= -f2; }
finish() {
cp "$DIR/root.crt" "$DIR/ca.crt" # public CA bundle for clients (make test CA_FILE=...)
cat "$DIR/root.crt" "$DIR/ingest.crt" > "$DIR/ingest-bundle.crt"
cat "$DIR/root.crt" "$DIR/telemetry.crt" > "$DIR/telemetry-bundle.crt"
touch "$DIR/revoked-serials.txt"
# Containers run as other uids and read keys through Docker secrets: files readable, directory closed.
chmod 700 secrets "$DIR"
chmod 644 "$DIR"/*.crt "$DIR"/*.key "$DIR/revoked-serials.txt"
chmod 600 "$DIR"/*.previous 2>/dev/null || true
chmod 600 "$DIR"/root.key "$DIR"/ingest.key "$DIR"/telemetry.key "$DIR"/foreign.key
}
cmd="${1:-init}"
mkdir -p "$DIR"
case "$cmd" in
init)
ca_init root "/O=Messenger Lab/CN=Messenger Lab Root CA"
ca_init ingest "/O=Messenger Lab/CN=Messenger Telemetry Ingest CA" root
ca_init telemetry "/O=Messenger Lab/CN=Messenger Telemetry Clients CA" root
for name in $(echo "$LEAVES" | awk 'NF {print $1}'); do
[ -f "$DIR/$name.crt" ] || issue "$name"
done
foreign
finish ;;
issue) issue "$2"; finish ;;
revoke)
s=$(serial "$2"); grep -qx "$s" "$DIR/revoked-serials.txt" 2>/dev/null || echo "$s" >> "$DIR/revoked-serials.txt"
echo "revoked $2 serial=$s"; finish ;;
revoke-serial)
s=$(echo "$2" | tr 'a-f' 'A-F')
# append in place: the file is bind-mounted into the API containers (a new inode would not be seen)
grep -qx "$s" "$DIR/revoked-serials.txt" 2>/dev/null || echo "$s" >> "$DIR/revoked-serials.txt"
echo "revoked serial=$s" ;;
expired) expired "$2"; finish ;;
serial) serial "$2" ;;
list)
for name in $(echo "$LEAVES" | awk 'NF {print $1}'); do
printf '%-15s %-28s %s %s\n' "$name" "$(leaf_line "$name" | awk '{print $4}')" "$(serial "$name")" \
"$(docker run --rm -v "$PWD/$DIR:/pki" -w /pki "$IMAGE" x509 -in "$name.crt" -noout -enddate | cut -d= -f2)"
done ;;
*) echo "usage: $0 init|issue <leaf>|revoke <leaf>|revoke-serial <hex>|expired <leaf>|serial <leaf>|list" >&2; exit 2 ;;
esac
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
try:
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
finally:
await db.close()
if __name__ == "__main__":
asyncio.run(main())
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
# Restart hook (tests/resilience.py --mode persistence): restart every store keeping all volumes, then the services.
# business stores: postgres, valkey, minio, rabbitmq
# secret store: openbao -> comes back SEALED; the operator step `scripts/bao.sh unseal` uses 2 of 3 shares
# from secrets/operator/ (lab compromise documented in REPORT.md). Without it the services would
# wait for OpenBao after their own restart.
# telemetry: prometheus, loki, tempo, otel-collector, grafana
# services: worker, api-1, api-2, nginx (they log in to OpenBao again and re-read their secrets)
# Never removes volumes, never re-seeds. Prints no secrets.
set -eu
cd "$(dirname "$0")/.."
docker compose restart -t 10 postgres valkey minio rabbitmq openbao >/dev/null
sh scripts/bao.sh unseal >/dev/null
docker compose restart -t 10 prometheus loki tempo otel-collector grafana >/dev/null
docker compose restart -t 25 worker api-1 api-2 nginx >/dev/null
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
+105 -8
View File
@@ -1,19 +1,116 @@
# Модель угроз — заполнить
# Модель угроз — lab7
## Активы и акторы
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
**Активы:**
- пользовательские credentials: Argon2id-хеши паролей, Bearer-сессии (в БД — SHA-256 токена);
- сообщения и вложения: PostgreSQL и MinIO;
- service identities: TLS-ключи `lb`, `api`, `worker`, `api-otel`, `worker-otel`, `collector-out`, `grafana-out`, `operator`;
- закрытые ключи CA, root token и доли unseal OpenBao;
- секреты сервисов в OpenBao: пароли ролей БД, ключи S3, пароль Valkey;
- telemetry: логи, трассы, метрики.
**Акторы:**
| Актор | Возможности в модели |
| --- | --- |
| Участник чата | легитимный пользователь с Bearer-токеном |
| Посторонний пользователь | зарегистрирован, но не состоит в чате; знает или угадывает UUID чужих объектов |
| Внешний клиент | доступ только к `127.0.0.1:8443`, может перебирать пароли и подделывать заголовки |
| Скомпрометированный сервис | атакующий исполняет код в одном контейнере (worker, API, Grafana) и получает его ключи и сетевую позицию во внутренней сети |
| Посторонний контейнер в сети | любой процесс в сети Compose без выданных нами ключей, либо с сертификатом чужого CA |
| Оператор | доверенный: держит ключи CA, root token и доли unseal |
**Границы стенда:**
- один Docker-host, сеть Compose считается враждебной: все связи идут по TLS;
- компрометация самого хоста и Docker daemon вне модели (root на хосте читает всё);
- данные синтетические.
## Границы доверия
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
```
внешний клиент ──TLS──► nginx ──mTLS (lb identity, SAN allowlist)──► api-1, api-2
│ TLS verify-full / HTTPS / AMQPS mTLS / TLS+ACL
worker ─────────────────────────────┼──► PostgreSQL, MinIO, RabbitMQ, Valkey
api, worker ──HTTPS + cert auth──► OpenBao (KV: свой путь, токен 15 мин)
api, worker ──mTLS (Ingest CA)──► Collector ──mTLS (Telemetry Clients CA)──► Prometheus / Loki / Tempo
Grafana ──mTLS (Telemetry Clients CA)──┘
оператор ──docker compose exec / 127.0.0.1 (HTTPS, mTLS)──► OpenBao, Grafana, Prometheus/Loki/Tempo
```
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
| --- | --- | --- | --- | --- |
| TODO | TODO | TODO | TODO | TODO |
Полная матрица со всеми портами, CA, identities и тестами: [contracts/trust-matrix.md](../contracts/trust-matrix.md).
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
## Угрозы, контроли и проверки
| # | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
| --- | --- | --- | --- | --- | --- |
| T1 | Скомпрометированный worker или посторонний контейнер подключается к `api-1:8080` напрямую и вызывает бизнес-API в обход nginx, подделывая `X-Real-IP` | к API ходит только LB; клиентский IP для лимитов достоверен | mTLS CERT_REQUIRED + allowlist DNS SAN `lb.client.messenger` + проверка отозванных serial в `app/serve.py`; plaintext-listener нет; health только на 127.0.0.1 | аудит A: без сертификата, чужой CA с именем lb, `worker`/`api`/`intruder` от того же CA → соединение закрыто до HTTP; аудит K: 8081 закрыт | скомпрометированный nginx имеет полный доступ к API (он и есть LB) |
| T2 | Скомпрометированный worker читает пользователей, сессии и сообщения в БД, удаляет вложения, публикует поддельные задания или читает секрет API | worker видит только конвейер обработки | роль `app_worker` (SELECT/UPDATE jobs и attachments, INSERT variants/audit/outbox, без DELETE); S3-пользователь worker: GET `staging/*`, PUT `attachments/*`; RabbitMQ: только read очереди; OpenBao: только `messenger/worker` | аудит B: SELECT users/messages/sessions и DELETE → InsufficientPrivilege; C: GET/DELETE `attachments`, LIST, PUT `staging` → AccessDenied; D: publish → ACCESS_REFUSED; H: чтение `messenger/api` → 403 | worker может записать любые байты в `attachments/*` (подмена вариантов) и испортить статус заданий; это минимум, нужный для его работы |
| T3 | Украден ключ LB-клиента: атакующий во внутренней сети представляется LB | украденная identity отзывается без пересборки и не принимается | короткий срок leaf (90 дн.), `pki.sh issue` + `nginx -s reload`, `pki.sh revoke-serial` → API отклоняет старый serial через ~1 с; просроченные сертификаты отклоняет TLS | `test_cert_rotation.py`: после отзыва старый сертификат закрыт на обеих репликах (`reason=revoked`), новый работает, просроченный отклонён; 0 ошибок трафика | до отзыва украденный ключ действует; отзыв проверяет только API |
| T4 | Скомпрометированный сервис с сертификатом от того же Root CA (например, LB или api) пишет поддельную telemetry напрямую в Prometheus/Loki/Tempo, читает чужие логи или обходит Collector | ingest только через Collector и только от приложений; хранилища читают только collector, grafana и оператор | отдельные промежуточные CA: Collector доверяет только Ingest CA, бэкенды — только Telemetry Clients CA | аудит E: `lb`, `api` (Root), `grafana-out` → отклонены Collector; аудит F: `api`, `api-otel` → отклонены Prometheus/Loki/Tempo | Collector не связывает `service.name` с сертификатом: `api-otel` может выдать себя за worker в telemetry |
| T5 | Внешний клиент перебирает пароли (credential stuffing) через обе реплики: каждая реплика считает попытки отдельно | ограничение действует на систему целиком | счётчики в Valkey, общие для реплик: 5 неудачных попыток на username и 120 попыток на IP за 60 с → 429 + Retry-After, до вычисления Argon2 | `test_auth_acl.py`: 5 неудач через `api-1` и `api-2` → 429 даже с верным паролем; 107 попыток с одного IP → 429 | распределённый перебор с многих IP ограничен только лимитом на username; при недоступном Valkey работает локальный (не общий) лимит |
| T6 | DoS через дорогой вход: поток Basic-запросов на несуществующих пользователей загружает CPU и RAM Argon2id | обычные запросы продолжают обслуживаться, память ограничена | не больше 4 одновременных Argon2 на реплику (иначе 503 через 3 с); dummy-хеш для неизвестных пользователей; лимиты контейнера 0.5 CPU / 512 MiB | `login_budget.py`: 32 атакующих, 40 попыток/с: CPU API в лимите, RSS 97 → ~190 MiB, обычные запросы без ошибок (p99 46 → 295 мс) | атака снижает latency; полный DDoS вне модели (нужен лимит на ingress) |
| T7 | Посторонний пользователь читает чужой чат, историю, метаданные или содержимое вложения по UUID, прикрепляет чужой файл к своему сообщению, подменяет `X-User-Id` | доступ только участникам; ответ не раскрывает существование | проверка членства на каждом запросе, единый 404; identity только из Bearer | `test_auth_acl.py`: 404 на chat/history/metadata/content/variant/post/attach; поддельный `X-User-Id` не меняет identity; Basic на бизнес-endpoint → 401 | — |
| T8 | Украденный Bearer-токен используется после logout или истечения срока | отозванная или истёкшая сессия не принимается ни одной репликой | серверные сессии (SHA-256 токена в БД), отзыв в Valkey и БД, срок в кэше не дольше `expires_at` | `test_auth_acl.py`: после logout и после истечения → 401 на обеих репликах | принудительное изменение срока в БД действует через ≤ 30 с (TTL кэша); logout действует сразу |
| T9 | Утечка пароля роли worker (дамп env или лог) | утёкший пароль быстро становится бесполезным | пароль только в OpenBao, не в env, файлах или образе; `bao.sh rotate-worker-db`; worker подхватывает новую версию KV без рестарта | `test_secret_rotation.py`: старый пароль отклонён, новые соединения используют новый, задания продолжаются | открытые до ротации соединения живут до `max_lifetime` (600 с) или принудительного завершения |
| T10 | OpenBao запечатан или упал (рестарт, атака), и сервисы стартуют с пустыми credentials или с ослабленной проверкой | секреты не подменяются ничем слабым | запущенные процессы держат полученные секреты; новый процесс ждёт до 120 с и завершается с кодом 78; распечатывает только оператор | `test_bao_sealed.py`: smoke проходит в запечатанном состоянии, перезапущенный worker ждёт и не становится healthy, после unseal восстанавливается | при длительном запечатывании токены истекают (≤ 1 ч), ротация не подхватывается; нужен оператор |
| T11 | MITM или подмена хранилища во внутренней сети: посторонний контейнер выдаёт себя за `postgres`, `minio`, `rabbitmq` или `valkey`, либо клиент откатывается на plaintext | каждый клиент проверяет сервер, plaintext не принимается | verify-full / verify=CA / ssl_check_hostname у всех клиентов; plaintext-порты закрыты (`listeners.tcp = none`, Valkey `--port 0`, `pg_hba` reject) | аудит B/C/G/I: чужой CA как trust anchor, неверный hostname → ошибка проверки; plaintext → отказ или закрытие | — |
| T12 | Секреты или содержимое утекают в telemetry: пароль, токен, текст сообщения, filename, байты изображения | telemetry не содержит секретов и пользовательского контента | захват заголовков выключен, View ограничивает атрибуты, SQL параметризован, collector `attributes/scrub` удаляет `url.query` и заголовки | `test_observability.py`: пароль, токен, маркер текста, filename, `Authorization`, signed URL, PNG (сырой и base64) в трассах и логах не найдены | технические UUID есть в span-атрибутах (обосновано в lab6) |
| T13 | Выход из контейнера или повышение привилегий через уязвимость приложения | ограниченный радиус поражения | non-root во всех 15 контейнерах, `cap_drop: ALL`, `no-new-privileges`, read-only rootfs + tmpfs, лимиты CPU/RAM/PIDs, нет Docker socket и host network | `check_hardening.py`: 15 контейнеров, 0 проблем (по `docker inspect`) | ядро хоста общее; seccomp и AppArmor по умолчанию Docker |
| T14 | Вредоносное изображение эксплуатирует уязвимость декодера (Pillow) | разбор недоверенных данных изолирован и сделан на актуальной версии | Pillow 13.0.0 (было 11.0.0 с 39 HIGH/CRITICAL CVE); декодирование в дочернем процессе с RLIMIT_AS/CPU и таймаутом; проверка magic и пикселей до декодирования | `variants_check.py` (лимиты пикселей и памяти дочернего процесса); `evidence/image_scan.txt`: CRITICAL Python 5 → 0 | новые CVE в Pillow появятся: нужен регулярный scan |
## Lifecycle и инциденты
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
**Bootstrap trust:**
1. Оператор создаёт CA и leaf-сертификаты (`scripts/pki.sh init`) и bootstrap-пароли (`scripts/init-secrets.sh`) — только в `secrets/` (0700, в `.gitignore` и `.dockerignore`).
2. Оператор инициализирует OpenBao (`scripts/bao.sh init`, 3 доли, порог 2) и записывает политики, роли cert auth и секреты (`bao.sh bootstrap`).
3. Первичная identity сервиса — его TLS-ключ, выданный оператором как Docker secret. Токена, SecretID и паролей в образе, `.env` или переменных окружения нет. В OpenBao сервис входит этим сертификатом и получает токен на 15 минут с правом чтения только своего пути.
**Renewal:**
- токен продлевается (renew-self) или запрашивается заново каждые 15 с;
- leaf-сертификаты живут 90 дней, перевыпуск — `pki.sh issue <leaf>`. Файлы переписываются на месте, nginx достаточно `nginx -s reload`, API/worker — перезапуска контейнера без пересборки образа.
**Revoke и rotation:**
- клиентский сертификат LB — `pki.sh revoke-serial`, API отклоняет его через ~1 с;
- пароль роли worker — `bao.sh rotate-worker-db`, worker подхватывает новую версию KV без рестарта;
- identity в RabbitMQ — удалить пользователя из `definitions.json`;
- в OpenBao — удалить роль `auth/cert/certs/<role>`.
**Компрометация identity** (пример — worker):
1. Отозвать роль worker в OpenBao, удалить пользователя `worker.client.messenger` в RabbitMQ.
2. Ротировать пароль `app_worker` и ключ S3 worker.
3. Выпустить новый сертификат worker.
4. Задания сохраняются в PostgreSQL и outbox и будут обработаны после восстановления.
**Запечатанный OpenBao:**
- запущенные сервисы работают дальше на полученных секретах;
- перезапущенный сервис ждёт (`openbao_not_ready`) и не становится healthy;
- восстановление: `scripts/bao.sh unseal` (2 из 3 долей).
**Владельцы:** CA, root token и доли unseal принадлежат оператору. В учебном стенде всё это у одного человека на одном хосте в `secrets/operator/` (0600). В production доли раздаются разным людям, а root token отзывается после bootstrap.
## Scan и применимость
Trivy 0.58.1. Образ передаётся сканеру через `docker save`, без Docker socket. Сводка: [evidence/image_scan.txt](../evidence/image_scan.txt).
**Образ приложения: до → после.**
- Базовый образ `python:3.12.8-slim` → `3.12.16-slim`, Pillow 11.0.0 → 13.0.0, urllib3 2.3.0 → 2.8.0.
- ОС: CRITICAL 9 → 2, HIGH/CRITICAL с доступным исправлением 56 → 0.
- Python: CRITICAL 5 → 0, HIGH 43 → 3.
- Секретов в слоях не найдено.
**Остаток:**
| Находка | Применимость | Решение |
| --- | --- | --- |
| zlib CVE-2023-45853 (CRITICAL, исправления нет) | MiniZip (zip-архивы) не используется; Python `zlib` в `imagecheck.py` — только `inflate` для PNG | принято, следить за обновлением |
| sqlite CVE-2025-7458 (CRITICAL, исправления нет) | нужен произвольный SQL к SQLite; SQLite в приложении не используется | принято |
| Kerberos (libkrb5*), util-linux — HIGH без исправления | GSSAPI в psycopg не используется (SCRAM по TLS); util-linux — инструменты образа, не код приложения | принято |
| starlette CVE-2025-62727 (Range в FileResponse) | `FileResponse` и `StaticFiles` не используются | неприменимо |
| starlette CVE-2026-48818 (StaticFiles на Windows) | Linux, `StaticFiles` нет | неприменимо |
| starlette CVE-2026-54283 (лимиты `request.form()`) | `form()` и multipart не используются: JSON и сырой upload с собственными лимитами 64 KiB / 10 MiB | неприменимо |
| starlette CVE-2026-48710 / 54282 (`request.url` собирается из Host и пути) | частично: guard использовал `request.url.path` для освобождения health-путей от проверок готовности | **исправлено**: везде `request.scope["path"]` (сырой путь маршрутизации) |
| nginx 1.27.3 (CRITICAL 3, исправления есть) | ingress | **обновлено** до 1.30.6-alpine |
| OpenBao 2.1.0 (Go stdlib CVE в TLS-сервере, исправления есть) | хранилище секретов | **обновлено** до 2.6.5; хранилище `file` подхвачено без миграции |
Starlette не обновлялась: для этого нужен мажорный переход FastAPI 0.115 → 0.16x. Это следующий шаг, но не в рамках лабораторной, потому что применимых уязвимостей не осталось.
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Every Prometheus query of the provisioned dashboard must parse and return data (after some traffic).
python3 student-tests/check_dashboard.py
"""
import json
import sys
import urllib.error
from pathlib import Path
import obs
ROOT = Path(__file__).resolve().parents[1]
dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8"))
bad = 0
for panel in dash["panels"]:
for t in panel.get("targets", []):
if t.get("datasource", {}).get("type") != "prometheus":
continue
expr = t["expr"].replace("$__rate_interval", "2m")
try:
rows = obs.prom(expr)
status = "ok %3d series" % len(rows) if rows else "EMPTY"
except urllib.error.HTTPError as exc:
status, rows = "ERROR %s" % exc.read()[:120], None
if not rows:
bad += 1
print("%-16s %-48s %s" % (status, panel["title"][:48], t.get("legendFormat", "")))
print("panels: %d, prometheus queries without data: %d" % (len(dash["panels"]), bad))
# ---------------------------------------------------------------- the same panels executed BY GRAFANA (/api/ds/query)
# Proves the provisioned datasources work from Grafana itself (Prometheus, Loki and Tempo), with Grafana macros.
import base64 # noqa: E402
import urllib.request # noqa: E402
password = (ROOT / "secrets/bootstrap/grafana_admin_password").read_text().strip()
import ssl # noqa: E402
GRAFANA_TLS = ssl.create_default_context(cafile=str(ROOT / "secrets/pki/ca.crt")) # lab7: Grafana is HTTPS-only
auth = "Basic " + base64.b64encode(("admin:" + password).encode()).decode()
via_bad = 0
for panel in dash["panels"]:
for t in panel.get("targets", []):
q = dict(t)
q.setdefault("datasource", panel.get("datasource"))
if q["datasource"]["type"] == "tempo":
# Grafana 11 runs TraceQL search in the browser through the datasource proxy, not /api/ds/query
# ("backend TraceQL search queries are not supported"): check exactly that proxy path.
import time as _t
import urllib.parse as _p
url = ("https://localhost:3000/api/datasources/proxy/uid/tempo/api/search?" +
_p.urlencode({"q": q["query"], "limit": q.get("limit", 20), "start": int(_t.time()) - 1800,
"end": int(_t.time())}))
req = urllib.request.Request(url, headers={"Authorization": auth})
try:
with urllib.request.urlopen(req, timeout=60, context=GRAFANA_TLS) as response:
n = len(json.loads(response.read()).get("traces", []))
status = "ok %5d traces" % n if n else "EMPTY"
except urllib.error.HTTPError as exc:
status = "ERROR HTTP %s" % exc.code
if not status.startswith("ok"):
via_bad += 1
print("grafana %-8s %-18s %-48s (datasource proxy)" % ("tempo", status, panel["title"][:48]))
continue
body = json.dumps({"queries": [q], "from": "now-30m", "to": "now"}).encode()
req = urllib.request.Request("https://localhost:3000/api/ds/query", data=body, method="POST",
headers={"Content-Type": "application/json", "Authorization": auth})
try:
with urllib.request.urlopen(req, timeout=60, context=GRAFANA_TLS) as response:
res = json.loads(response.read())["results"][q["refId"]]
frames = res.get("frames", [])
rows = sum(len((f.get("data") or {}).get("values", [[]])[0] or []) for f in frames)
status = "ERROR %s" % res["error"][:100] if res.get("error") else ("ok %5d values" % rows if rows else "EMPTY")
except urllib.error.HTTPError as exc:
status = "ERROR HTTP %s %s" % (exc.code, exc.read()[:100])
if not status.startswith("ok"):
via_bad += 1
print("grafana %-8s %-18s %-48s" % (q["datasource"]["type"], status, panel["title"][:48]))
print("via Grafana: queries without data or with errors: %d" % via_bad)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Level 3-4 hardening audit from the REAL container state (docker inspect), not from compose.yaml.
Checks every running container of the project: non-root user, not privileged, no host network/PID/IPC, no Docker
socket mount, all capabilities dropped (no cap_add), no-new-privileges, read-only root FS (+ tmpfs list), CPU/memory/
PIDs limits, and that only the allowed ports are published, and only on 127.0.0.1.
python3 student-tests/check_hardening.py [--output evidence/hardening.json]
"""
import argparse
import json
import subprocess
import sys
ALLOWED_PUBLISHED = {"8443": "nginx business ingress (TLS)", "3000": "Grafana (HTTPS + login)",
"9090": "Prometheus (mTLS operator)", "3100": "Loki (mTLS operator)", "3200": "Tempo (mTLS operator)"}
def main():
p = argparse.ArgumentParser()
p.add_argument("--output", default="")
a = p.parse_args()
ids = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab7"],
capture_output=True, text=True).stdout.split()
rows, failures = [], []
for info in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout):
name = info["Name"].strip("/").replace("mtusi-lab7-", "")
hc, cfg = info["HostConfig"], info["Config"]
user = cfg.get("User") or "root(image default)"
mounts = [m.get("Source", "") for m in info["Mounts"]]
published = []
for port, binds in (info["NetworkSettings"]["Ports"] or {}).items():
for b in binds or []:
published.append("%s:%s->%s" % (b["HostIp"], b["HostPort"], port))
row = {
"container": name, "user": user, "privileged": hc["Privileged"], "network": hc["NetworkMode"],
"pid": hc.get("PidMode") or "private", "cap_drop": hc.get("CapDrop"), "cap_add": hc.get("CapAdd"),
"no_new_privileges": "no-new-privileges:true" in (hc.get("SecurityOpt") or []),
"read_only_rootfs": hc["ReadonlyRootfs"], "tmpfs": sorted((hc.get("Tmpfs") or {}).keys()),
"cpus": hc["NanoCpus"] / 1e9, "memory_mib": hc["Memory"] // 1048576, "pids_limit": hc.get("PidsLimit"),
"docker_socket": any("docker.sock" in m for m in mounts), "published": published,
}
problems = []
if user.split(":")[0] in ("", "0", "root", "root(image default)"):
problems.append("runs as root")
if row["privileged"] or row["network"] == "host" or row["pid"] == "host" or row["docker_socket"]:
problems.append("privileged/host namespace/docker socket")
if row["cap_drop"] != ["ALL"] or row["cap_add"]:
problems.append("capabilities not fully dropped")
if not row["no_new_privileges"]:
problems.append("no-new-privileges missing")
if not row["read_only_rootfs"]:
problems.append("writable root FS")
if not (row["cpus"] and row["memory_mib"] and row["pids_limit"]):
problems.append("missing CPU/memory/PIDs limit")
for pub in published:
host_ip, rest = pub.split(":", 1)
if host_ip != "127.0.0.1" or rest.split("->")[0] not in ALLOWED_PUBLISHED:
problems.append("unexpected published port " + pub)
row["problems"] = problems
rows.append(row)
failures += ["%s: %s" % (name, x) for x in problems]
rows.sort(key=lambda r: r["container"])
print("%-16s %-11s %-6s %-5s %-4s %-5s %-9s %-24s %s" % ("container", "user", "capDrop", "nnp", "ro", "cpus",
"mem/pids", "published", "problems"))
for r in rows:
print("%-16s %-11s %-6s %-5s %-4s %-5s %-9s %-24s %s" % (
r["container"][:16], r["user"][:11], "ALL" if r["cap_drop"] == ["ALL"] else r["cap_drop"],
r["no_new_privileges"], r["read_only_rootfs"], r["cpus"], "%d/%s" % (r["memory_mib"], r["pids_limit"]),
",".join(x.split("->")[0] for x in r["published"]) or "-", "; ".join(r["problems"]) or "ok"))
print("published ports allowed: %s" % ALLOWED_PUBLISHED)
print("%d containers, %d problems" % (len(rows), len(failures)))
if a.output:
with open(a.output, "w", encoding="utf-8") as handle:
json.dump(rows, handle, ensure_ascii=False, indent=1)
sys.exit(1 if failures else 0)
if __name__ == "__main__":
main()
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Argon2id parameters and the login DoS budget, measured on the stand (level 3 + level 4).
bench inside api-1 (same CPU limit 0.5 core as production replicas): time and peak RSS of one hash for
several parameter sets -> justification of m=19 MiB, t=2, p=1
flood N concurrent attackers send wrong-password logins (distinct usernames, so per-user limits do not
help; the per-IP limit must be raised for this run to model a distributed attacker) while a probe
client sends normal requests; reports login throughput, 503 hash_busy share, API CPU/RSS and probe
latency vs. a baseline without the flood.
LOGIN_RATE_LIMIT_PER_IP=1000000 docker compose --env-file .env up -d --no-deps api-1 api-2 # experiment only
python3 student-tests/login_budget.py --output evidence/login_budget.json
docker compose --env-file .env up -d --no-deps api-1 api-2 # restore limits
"""
import argparse
import base64
import http.client
import json
import ssl
import statistics
import subprocess
import threading
import time
import uuid
from pathlib import Path
from stand import ROOT, Client, log
BENCH = r'''
import json, resource, time
from argon2 import PasswordHasher
out = []
for m, t, p in ((19456, 2, 1), (12288, 3, 1), (47104, 1, 1), (65536, 3, 1), (65536, 3, 4)):
h = PasswordHasher(time_cost=t, memory_cost=m, parallelism=p)
h.hash("warm-up-password")
times = []
for _ in range(10):
t0 = time.perf_counter(); h.hash("benchmark-password-123"); times.append((time.perf_counter() - t0) * 1000)
times.sort()
out.append({"m_kib": m, "t": t, "p": p, "p50_ms": round(times[5], 1), "max_ms": round(times[-1], 1),
"peak_rss_mib": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1)})
print(json.dumps(out))
'''
def ctx():
return ssl.create_default_context(cafile=str(ROOT / "secrets/pki/ca.crt"))
def request(conn, method, path, headers, body=None):
conn.request(method, path, body=body, headers=headers)
r = conn.getresponse()
r.read()
return r.status
def cgroup_cpu():
"""CPU seconds of api-1 + api-2 from the Docker API (exact over a window)."""
host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"],
capture_output=True, text=True).stdout.strip().replace("unix://", "")
total = 0.0
for name in ("mtusi-lab7-api-1-1", "mtusi-lab7-api-2-1"):
raw = subprocess.run(["curl", "-s", "--unix-socket", host,
"http://localhost/containers/%s/stats?stream=false&one-shot=true" % name],
capture_output=True, text=True).stdout
total += json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9
return total
def max_rss():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
capture_output=True, text=True).stdout
return {l.split()[0]: l.split()[1] for l in out.splitlines() if "api-" in l and "lab7" in l}
def probe(actor, chat, stop, lat):
conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30)
headers = {"Authorization": "Bearer " + actor["token"], "Content-Type": "application/json"}
while not stop.is_set():
t0 = time.perf_counter()
try:
h = dict(headers, **{"Idempotency-Key": uuid.uuid4().hex})
status = request(conn, "POST", "/api/v1/chats/%s/messages" % chat["id"], h, b'{"text":"probe"}')
lat.append((time.perf_counter() - t0, status))
except (OSError, http.client.HTTPException):
conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30)
time.sleep(0.2)
def attacker(stop, results):
conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30)
while not stop.is_set():
name = "nouser_" + uuid.uuid4().hex[:12]
auth = base64.b64encode((name + ":wrong-password-123").encode()).decode()
t0 = time.perf_counter()
try:
status = request(conn, "POST", "/api/v1/auth/sessions", {"Authorization": "Basic " + auth})
except (OSError, http.client.HTTPException):
conn = http.client.HTTPSConnection("localhost", 8443, context=ctx(), timeout=30)
status = "conn_error"
results.append((time.perf_counter() - t0, status))
def phase(actor, chat, attackers, duration):
stop, lat, logins = threading.Event(), [], []
threads = [threading.Thread(target=probe, args=(actor, chat, stop, lat))]
threads += [threading.Thread(target=attacker, args=(stop, logins)) for _ in range(attackers)]
cpu0, t0 = cgroup_cpu(), time.time()
for t in threads:
t.start()
rss = []
while time.time() - t0 < duration:
time.sleep(5)
rss.append(max_rss())
stop.set()
for t in threads:
t.join(40)
elapsed = time.time() - t0
cpu = (cgroup_cpu() - cpu0) / elapsed * 100
ok = sorted(x for x, s in lat if s == 201)
codes = {}
for _, s in logins:
codes[str(s)] = codes.get(str(s), 0) + 1
login_lat = sorted(x for x, s in logins if s == 401)
pct = lambda v, q: round(v[min(len(v) - 1, int(q * len(v)))] * 1000, 1) if v else None # noqa: E731
return {"attackers": attackers, "seconds": round(elapsed), "login_attempts_per_s": round(len(logins) / elapsed, 1),
"login_status_counts": codes, "login_401_p50_ms": pct(login_lat, .5), "login_401_p99_ms": pct(login_lat, .99),
"probe_ok": len(ok), "probe_errors": sum(1 for _, s in lat if s != 201),
"probe_p50_ms": pct(ok, .5), "probe_p99_ms": pct(ok, .99),
"api_cpu_pct_of_one_core_both_replicas": round(cpu, 1), "api_rss_samples": rss[-1] if rss else {}}
def main():
p = argparse.ArgumentParser()
p.add_argument("--duration", type=int, default=30)
p.add_argument("--attackers", type=int, default=32)
p.add_argument("--output", default=str(ROOT / "evidence/login_budget.json"))
a = p.parse_args()
bench = json.loads(subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", BENCH], cwd=ROOT,
capture_output=True, text=True, check=True).stdout)
for row in bench:
log("argon2id m=%(m_kib)d KiB t=%(t)d p=%(p)d: p50 %(p50_ms)s ms, max %(max_ms)s ms, peak RSS %(peak_rss_mib)s MiB" % row)
c = Client()
alice, _bob, _eve, chat = c.bootstrap()
base = phase(alice, chat, 0, a.duration)
log("baseline: %s" % base)
flood = phase(alice, chat, a.attackers, a.duration)
log("flood: %s" % flood)
result = {"argon2_bench_api_container_0_5_cpu": bench, "baseline": base, "flood": flood,
"config": {"PASSWORD_HASH_CONCURRENCY_per_replica": 4, "PASSWORD_HASH_QUEUE_SECONDS": 3,
"replicas": 2, "api_cpu_limit": 0.5, "api_mem_limit_mib": 512}}
Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result, ensure_ascii=False, indent=1))
if __name__ == "__main__":
main()
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env python3
"""In-network probe: GET /health/ready on an API replica with a given client certificate from /pki.
docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./student-tests:/audit:ro --entrypoint python \
worker /audit/mtls_probe.py api-1 lb-client [lb-client-old ...] -> one JSON line per certificate
"""
import json
import sys
sys.path.insert(0, "/audit")
import tls_audit # noqa: E402
host = sys.argv[1]
for name in sys.argv[2:]:
verdict, detail = tls_audit.http_get(host, 8080, "/health/ready", name)
print(json.dumps({"host": host, "cert": name, "verdict": verdict, "detail": detail}))
+146
View File
@@ -0,0 +1,146 @@
"""Query helpers for the telemetry backends (stdlib only). Backends listen on 127.0.0.1 (compose.yaml).
Never prints tokens or passwords; callers print only ids, counts, span names and attributes.
"""
import base64
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
import ssl
from pathlib import Path
# lab7: the backends accept only clients of the "Telemetry Clients CA": evidence scripts use the operator cert.
PROM = os.environ.get("PROM_URL", "https://localhost:9090")
LOKI = os.environ.get("LOKI_URL", "https://localhost:3100")
TEMPO = os.environ.get("TEMPO_URL", "https://localhost:3200")
_PKI = Path(__file__).resolve().parents[1] / "secrets/pki"
TLS = ssl.create_default_context(cafile=str(_PKI / "ca.crt"))
TLS.load_cert_chain(str(_PKI / "operator.crt"), str(_PKI / "operator.key"))
def _get(url, params=None, timeout=15):
if params:
url += "?" + urllib.parse.urlencode(params)
with urllib.request.urlopen(url, timeout=timeout, context=TLS) as response:
return json.loads(response.read())
def prom(expr, at=None):
"""Instant query -> list of (labels, float value)."""
params = {"query": expr}
if at:
params["time"] = at
data = _get(PROM + "/api/v1/query", params)["data"]["result"]
return [(r["metric"], float(r["value"][1])) for r in data]
def prom_scalar(expr, default=0.0, at=None):
rows = prom(expr, at)
return rows[0][1] if rows else default
def prom_range(expr, start, end, step=5):
data = _get(PROM + "/api/v1/query_range", {"query": expr, "start": start, "end": end, "step": step})
return data["data"]["result"]
def alerts():
"""{alertname: state} for pending/firing alerts in Prometheus."""
out = {}
for a in _get(PROM + "/api/v1/alerts")["data"]["alerts"]:
out[a["labels"]["alertname"]] = a["state"]
return out
def loki(query, minutes=15, limit=500):
end = time.time_ns()
data = _get(LOKI + "/loki/api/v1/query_range", {"query": query, "limit": limit, "direction": "forward",
"start": end - minutes * 60 * 10**9, "end": end})
lines = []
for stream in data["data"]["result"]:
for ts, line in stream["values"]:
lines.append({"ts": int(ts), "line": line, **stream["stream"]})
return sorted(lines, key=lambda item: item["ts"])
def tempo_search(traceql, minutes=30, limit=100, start=None, end=None):
end = int(end or time.time() + 60)
start = int(start or end - 60 - minutes * 60)
data = _get(TEMPO + "/api/search", {"q": traceql, "start": start, "end": end, "limit": limit, "spss": 50},
timeout=60)
return data.get("traces", [])
def _hex(value):
return base64.b64decode(value).hex() if value else None
def _attr(value):
for key in ("stringValue", "intValue", "boolValue", "doubleValue"):
if key in value:
return value[key]
return None
def tempo_trace(trace_id):
"""Flat list of spans: service, name, kind, span_id, parent_id, duration_ms, status, attrs (dict)."""
try:
data = _get(TEMPO + "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return []
raise
spans = []
for batch in data.get("batches", []):
service = {a["key"]: _attr(a["value"]) for a in batch["resource"]["attributes"]}.get("service.name")
for scope in batch.get("scopeSpans", []):
for s in scope["spans"]:
spans.append({
"service": service, "name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""),
"span_id": _hex(s["spanId"]), "parent_id": _hex(s.get("parentSpanId")),
"start": int(s["startTimeUnixNano"]),
"duration_ms": (int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"])) / 1e6,
"status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", "") or "UNSET",
"attrs": {a["key"]: _attr(a["value"]) for a in s.get("attributes", [])},
})
return sorted(spans, key=lambda item: item["start"])
def tempo_raw(trace_id):
with urllib.request.urlopen(TEMPO + "/api/traces/" + trace_id, timeout=15, context=TLS) as response:
return response.read().decode("utf-8", "replace")
def print_tree(spans, out=print, skip_db=False):
children = {}
ids = {s["span_id"] for s in spans}
for s in spans:
children.setdefault(s["parent_id"] if s["parent_id"] in ids else None, []).append(s)
def walk(parent, depth):
for s in children.get(parent, []):
if skip_db and s["kind"] == "CLIENT" and s["name"].split()[0] in ("SELECT", "UPDATE", "INSERT", "DELETE"):
continue
extra = {k: v for k, v in s["attrs"].items() if k.startswith(("messenger.", "messaging.message", "db.system",
"http.response.status_code", "rpc.method"))}
out("%s%-8s %-9s %-44s %8.1f ms %-5s %s" % (" " * depth, s["service"].replace("messenger-", ""), s["kind"],
s["name"][:44], s["duration_ms"], s["status"],
json.dumps(extra, ensure_ascii=False)))
walk(s["span_id"], depth + 1)
walk(None, 0)
def wait_for(predicate, seconds=60, interval=2):
deadline = time.monotonic() + seconds
value = None
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(interval)
return value
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time,
so a slow server cannot slow down the generator (no coordinated omission while threads are available).
Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png).
Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500)
are added to GET history with the given percentages, to measure whether such traces survive sampling.
python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \
--output evidence/x.json
Tokens stay in memory; the summary contains counts, percentiles and timestamps only.
"""
import argparse
import http.client
import json
import os
import queue
import random
import ssl
import sys
import threading
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
os.environ.setdefault("CA_FILE", str(ROOT / "secrets/pki/ca.crt"))
from client import Client # noqa: E402
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def percentile(values, p):
if not values:
return None
values = sorted(values)
k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1))
return values[k]
class Load:
def __init__(self, base_url, users=4, threads=64, timeout=30.0):
self.base = urlsplit(base_url)
self.tls = ssl.create_default_context(cafile=os.environ.get("CA_FILE") or str(ROOT / "secrets/pki/ca.crt"))
self.threads = threads
self.timeout = timeout
c = Client(base_url)
self.actors = []
for i in range(users):
a, b = c.user("load%da" % i), c.user("load%db" % i)
chat = c.call("POST", "/api/v1/chats", 201, actor=a,
body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"]
self.actors.append((a["token"], chat["id"]))
self.lock = threading.Lock()
self.results = []
self.local = threading.local()
def conn(self):
c = getattr(self.local, "conn", None)
if c is None:
if self.base.scheme == "https": # lab7: TLS verified against the stand CA, never unverified
c = http.client.HTTPSConnection(self.base.hostname, self.base.port or 443, timeout=self.timeout,
context=self.tls)
else:
c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout)
self.local.conn = c
return c
def request(self, method, path, body=None, headers=None):
for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending
conn = self.conn()
try:
conn.request(method, path, body=body, headers=headers or {})
response = conn.getresponse()
response.read()
return response.status, response.getheader("X-Request-Id")
except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc:
conn.close()
self.local.conn = None
if attempt == 2:
return "conn_error:" + type(exc).__name__, None
except (OSError, http.client.HTTPException) as exc:
conn.close()
self.local.conn = None
return "error:" + type(exc).__name__, None
return "error", None
def one(self, kind, fault, scheduled):
token, chat = random.choice(self.actors)
headers = {"Authorization": "Bearer " + token}
if fault:
headers["X-Fault"] = fault
if kind == "post_message":
headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex})
status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat,
json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers)
elif kind == "get_history":
status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers)
else:
headers["Content-Type"] = "image/png"
status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers)
done = time.monotonic()
with self.lock:
self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled,
"at": time.time(), "rid": rid})
def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None):
work = queue.Queue()
start = time.monotonic()
self.started_at = time.time()
def worker():
while True:
item = work.get()
if item is None:
return
self.one(*item)
pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)]
for t in pool:
t.start()
def schedule(r, picker):
if r <= 0:
return
n = 0
while True:
due = start + n / r
if due - start >= duration or (stop_event and stop_event.is_set()):
return
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
work.put(picker() + (due,))
n += 1
def pick_main():
if random.random() < 0.75:
return ("post_message", None)
roll = random.random() * 100
fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None
return ("get_history", fault) # 25% share * 4 = requested share of ALL requests
schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True),
threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)]
for t in schedulers:
t.start()
for t in schedulers:
t.join()
backlog = work.qsize()
for _ in pool:
work.put(None)
for t in pool:
t.join(self.timeout + 5)
self.finished_at = time.time()
return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog)
def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog):
res = self.results
ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400]
plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"]
out = {
"profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate,
"slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads,
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"},
"started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog,
"requests": len(res), "ok": len(ok),
"successful_rps": round(len(ok) / max(duration, 1e-9), 2),
"status_counts": {},
"server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and (
not isinstance(r["status"], int) or r["status"] >= 500)),
"latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None
for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))},
"faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"),
"error": sum(1 for r in res if r["fault"] == "error")},
"by_kind": {},
}
for r in res:
key = str(r["status"])
out["status_counts"][key] = out["status_counts"].get(key, 0) + 1
for kind in ("post_message", "get_history", "upload"):
lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None]
out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind),
"p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None,
"p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None}
return out
def main():
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--base-url", default="https://localhost:8443")
p.add_argument("--rate", type=float, default=40)
p.add_argument("--duration", type=float, default=60)
p.add_argument("--upload-rate", type=float, default=0)
p.add_argument("--slow-pct", type=float, default=0)
p.add_argument("--error-pct", type=float, default=0)
p.add_argument("--threads", type=int, default=64)
p.add_argument("--output")
a = p.parse_args()
load = Load(a.base_url, threads=a.threads)
summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct)
text = json.dumps(summary, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(a.output).write_text(text + "\n", encoding="utf-8")
if __name__ == "__main__":
main()

Some files were not shown because too many files have changed in this diff Show More