forked from mtusi2026/mtusi2026lab7
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5b28e7d130 | ||
|
|
8f82949094 |
@@ -17,4 +17,10 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
student-tests
|
||||
tests
|
||||
research
|
||||
REPORT.md
|
||||
SCHEMA.md
|
||||
requirements-dev.txt
|
||||
.DS_Store
|
||||
|
||||
@@ -39,3 +39,8 @@ BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
|
||||
PUBLIC_PORT=8443
|
||||
GRAFANA_PORT=3007
|
||||
TELEMETRY_ENABLED=1
|
||||
OTEL_SAMPLE_RATIO=1
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.14.7
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
|
||||
WORKDIR /app
|
||||
RUN python -m pip install --no-cache-dir uv==0.12.13
|
||||
ENV UV_PYTHON_DOWNLOADS=never
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
|
||||
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
|
||||
# The application uses the locked venv. The base image's unused pip vendors a separate CA bundle.
|
||||
RUN python -m pip uninstall --yes pip \
|
||||
&& rm -rf /usr/local/lib/python3.14/ensurepip \
|
||||
&& groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --no-create-home app
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY migrations /app/migrations
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
APP_LOG_LEVEL=INFO
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -31,3 +31,19 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down lint
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
|
||||
up: bootstrap
|
||||
docker compose up -d openbao
|
||||
$(PYTHON) scripts/bao_operator.py
|
||||
docker compose up -d --build --wait --wait-timeout 120
|
||||
|
||||
down:
|
||||
docker compose stop --timeout 7
|
||||
|
||||
lint:
|
||||
uv run --locked ruff check .
|
||||
uv run --locked ruff format --check .
|
||||
|
||||
@@ -1,37 +1,73 @@
|
||||
# Отчёт — лабораторная 7
|
||||
# ЛР7. Безопасность, mTLS и OpenBao
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Горбанюк, БВТ2301.** Целевая оценка 5. Трек 1 «Компрометация worker».
|
||||
|
||||
За основу взято собственное решение ЛР6. Использованы [OpenBao cert auth](https://openbao.org/docs/auth/cert/), [OpenBao 2.7.2](https://github.com/openbao/openbao/releases/tag/v2.7.2), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html) и материалы задания. При подготовке реализации, проверок и отчёта использовалась помощь ИИ.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Docker с Compose, OpenSSL 3 и Python 3.14 с `uv`. Сборка использует Python 3.14.7, `uv.lock`, HAProxy 3.2.26 и OpenBao 2.7.2. Остальные версии указаны в `compose.yaml`. Старт выполняется оператором:
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
make up
|
||||
export CA_FILE="$PWD/secrets/tls/trust/ca.crt"
|
||||
curl -fsS --cacert "$CA_FILE" https://localhost:8443/health/ready
|
||||
make test test-security
|
||||
uv sync --locked
|
||||
make check
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make lint
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
При первом запуске `scripts/bootstrap.py` создаёт локальные CA, leaf и credentials. `scripts/bao_operator.py` инициализирует одноузловой Raft, выполняет unseal и добавляет четыре политики сервиса. Закрытый ключ CA и JSON оператора хранятся в защищённом каталоге `secrets/operator/` с доступом только владельца, вне контейнеров. В учебном стенде один оператор и один unseal share. Для действующего хранилища нужно сохранить соответствующий файл оператора, а не создавать новый CA или новый volume при каждом запуске. Обычный перезапуск API проходит без участия оператора. После перезапуска OpenBao оператор выполняет `python3 scripts/bao_operator.py`.
|
||||
|
||||
## Доказательства критериев
|
||||
Опубликованы только loopback-порты 8443 (ingress), 3007 (Grafana) и 8207 (OpenBao). Grafana использует локального `admin`, пароль в `secrets/grafana_password`. Для запросов к Grafana применяется тот же учебный CA. `make down` останавливает стенд с данными. `docker compose down -v` намеренно удаляет volumes проекта, включая состояние OpenBao и прикладные данные.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
## Архитектура и границы
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Бизнес API, очередь и observability из ЛР6 сохранены. Пользователь работает через HTTPS и Bearer. HAProxy проверяет имя `api` и предъявляет отдельный сертификат; API допускает только URI SAN балансировщика. Раздельные сервисные клиенты используют TLS и проверку hostname. HAProxy перед внутренним broker и Collector проверяет сертификат и разрешённую роль, затем передаёт запрос на loopback в namespace компонента. Аналогичный подход защищает хранилища телеметрии, у источников Grafana отдельная identity и разрешение на чтение.
|
||||
|
||||
## Эксперименты и отказы
|
||||
Общие роли и проверки описаны в [матрице доверия](contracts/trust-matrix.md), модель атак — в [модели угроз](security/THREAT_MODEL.md). PostgreSQL содержит данные, сессии и общее состояние лимитера, S3 — файлы, Valkey Streams — доставку заданий. TLS и ACL PostgreSQL, S3 и брокера независимы. Worker не может прочитать пароли или сообщения из БД и не имеет доступа к API path OpenBao.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Cert auth OpenBao использует свой leaf для каждой роли, короткий токен и policy с чтением одного пути. SDK повторяет вход каждые 5 секунд и отзывает токен после чтения. После 30 секунд без подтверждения API отдаёт 503, relay и worker перестают принимать новые задачи. Начатый child ограничен worker timeout. При изменении секрета оператор пересоздаёт соответствующий сервис, чтобы закрыть старые соединения. Разрешение читать KV и разрешение подключаться к хранилищу проверяются отдельно.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Контейнеры приложения работают с UID 10001, `read_only`, `cap_drop: ALL`, `no-new-privileges` и ограниченными tmpfs, CPU/RAM/PID. OpenBao работает под UID 100 в обычном режиме. Тома БД, S3, брокера и telemetry пишутся состоянием соответствующего сервиса; их файловые права и место записи задаёт image/Compose. PostgreSQL entrypoint начинает с root для установки ключа с правами 0600, штатный postgres затем работает под своим UID. Admin jobs с root credentials короткоживущие и смонтированы отдельно от приложения. Сокет Docker и host network не используются.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Приёмка
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
```sh
|
||||
uv run python scripts/verify_sessions.py
|
||||
uv run python scripts/verify_telemetry.py
|
||||
uv run python scripts/security_probe.py --rotate --seal --output evidence/security-repeat.json
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose up -d --no-build --wait api lb
|
||||
```
|
||||
|
||||
- [Smoke](evidence/smoke.txt) и [HTTPS](evidence/https.txt) проверяют действительный CA, бизнес API, вложения и отказ недоверенному клиентскому trust. Отдельно проверено [несовпадение hostname](evidence/hostname.txt). [Чистый clone](evidence/clean-clone.txt) поднят в отдельном Compose project с новым Raft, volumes и сертификатами.
|
||||
- [Сессии](evidence/sessions.txt) проверяют миграцию старого Argon2id hash при успешном входе и отказ истёкшему токену. Новые параметры — 19 MiB, t=2, p=1. На реплику два KDF слота, то есть около 38 MiB под вычисление новых хэшей плюс память процесса; при проверке старого hash применяется его прежний, более высокий бюджет. Общий лимит 5 попыток на имя и 90 на стенд за минуту. В проверке одна учётная запись проходила через обе реплики и получила 429 с `Retry-After`. Ограничение JSON — 64 KiB, изображений — до 10 MiB и 40 млн пикселей.
|
||||
- [Телеметрия](evidence/telemetry.txt) проверяет доступ ко всем трём источникам через Grafana и проверяет лог и trace синтетической загрузки и обработки на отсутствие её тестовых credentials и содержимого.
|
||||
- [Результаты безопасности](evidence/security-final.json) фиксируют отрицательные TLS случаи для API, брокера и Collector, allow/deny для Grafana, OpenBao, DB и S3, ротацию и сценарий sealed.
|
||||
- [Перезапуск](evidence/persistence.txt) повторно поднимает API, PostgreSQL, S3, cache, broker и OpenBao с существующими volumes. [Failover](evidence/failover.txt) останавливает одну реплику по её instance ID, проверяет сохранность подтверждённых операций и работу оставшейся.
|
||||
- [Сканирование](evidence/scan.md) фиксирует версию базы уязвимостей, применимость выбранных находок и повторный результат после обновлений.
|
||||
|
||||
## Трек 1. Компрометация worker
|
||||
|
||||
Гипотеза: изолированные роли ограничивают доступ атакующего к контуру обработки, а отзыв identity и смена пароля брокера позволяют вернуть обработку подтверждённых задач.
|
||||
|
||||
В ЛР5 и ЛР6 worker подключался к общей роли БД и брокеру без отдельной ACL. В ЛР7 PostgreSQL допускает чтение очереди и ограниченные изменения состояния, отказал на чтение users/messages/sessions, создание TEMP table, смену владельца вложения и SET ROLE. Valkey разрешает протокол группы на конкретных ключах и отказал на чужой ключ и CONFIG GET. В S3 worker может прочитать staging и записать objects, получает отказ при попытке записи в staging, удаления и чтения bucket policy. Ограничения S3 расширяют уже существовавшее в ранней работе разделение ролей. OpenBao выдал токен только для своего пути и отказал в чтении пути API и записи. Явно отозванный токен получил отказ повторно.
|
||||
|
||||
Методика на одном Docker Desktop ARM64. Используются две реплики API, worker с двумя слотами, PostgreSQL и брокер с томами. В операторском сценарии останавливается worker и выполняется 5 загрузок одного PNG с ожиданием ответа 202. Старый сертификат сохраняется только в защищённой временной папке для отрицательной пробы. Оператор удаляет старую роль cert auth, отзывает leaf в CRL, выпускает новый leaf той же роли, меняет пароль worker в Valkey ACL и завершает старые соединения. TLS-прокси читают CRL после перезапуска, образы не пересобираются. Проба устанавливает новые подключения и подтверждает отказ старого leaf в broker, Collector и OpenBao. Затем operator setup привязывает новую identity, worker поднимается со свежим scoped secret.
|
||||
|
||||
Все 5 подтверждённых загрузок достигли ready, исходные изображения совпали по SHA-256. От начала пяти загрузок до проверки всех файлов прошло 8.47 с. Сценарий воспроизводится одним скриптом. При sealed API стал unready через 29.15 с. Готовность вернулась через 33.91 с от начала сценария. Измерения и последовательность сохранены в `evidence/security-final.json`.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Сертификат worker ограничивает доступ к OpenBao, Collector и broker. Уже извлечённые пароли БД и ключи S3 остаются действительными до ротации на соответствующем сервисе. Тесты показывают сохранность при отзыве и восстановлении, а не при произвольной вредоносной записи в разрешённый worker prefix. Система размещена на одном хосте; компрометация хоста, CA и оператора, обновление корневого CA без остановки и отказ диска выходят за пределы этого стенда. После замены namespace хранилища его TLS-прокси поднимается вместе с ним по Compose зависимости.
|
||||
|
||||
## Вопросы к защите
|
||||
|
||||
- **Что проверяет mTLS?** Цепочка и имя сервера нужны клиенту, сервер проверяет клиентскую цепочку и сопоставляет сертификат с разрешённой ролью. Дальше ACL решает, какие операции этой роли доступны.
|
||||
- **Почему доверенного CA мало?** Он может выпускать сертификаты для нескольких сервисов. Сертификат Grafana подписан тем же CA, но не даёт доступа к ingestion или broker.
|
||||
- **Как сервис впервые получает секрет?** Оператор выдаёт leaf и монтирует его только в сервис. Сертификат позволяет получить короткий токен на чтение своего KV path. Root token и unseal share остаются у оператора.
|
||||
- **Что происходит при истечении credentials?** Проверка identity и чтение повторяются. При длительной недоступности сервиса API перестаёт быть ready, новые задания не потребляются. После восстановления OpenBao и проверки секрета работа продолжается.
|
||||
|
||||
+862
@@ -0,0 +1,862 @@
|
||||
name: mtusi-lab7
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
- ./secrets/tls/postgres:/run/tls/server:ro
|
||||
- ./infra/security/postgres-entrypoint.sh:/security/postgres-entrypoint.sh:ro
|
||||
- ./infra/security/pg_hba.conf:/security/pg_hba.conf:ro
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- fsync=on
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- full_page_writes=on
|
||||
- -c
|
||||
- shared_preload_libraries=pg_stat_statements
|
||||
- -c
|
||||
- ssl=on
|
||||
- -c
|
||||
- ssl_min_protocol_version=TLSv1.2
|
||||
- -c
|
||||
- ssl_cert_file=/tmp/postgres-tls/server.crt
|
||||
- -c
|
||||
- ssl_key_file=/tmp/postgres-tls/server.key
|
||||
- -c
|
||||
- hba_file=/security/pg_hba.conf
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- pg_isready -h /var/run/postgresql -U messenger -d messenger
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: ${POSTGRES_CPU:-1}
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
networks:
|
||||
- backend
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- /security/postgres-entrypoint.sh
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- valkey-server
|
||||
- --save
|
||||
- ''
|
||||
- --appendonly
|
||||
- 'no'
|
||||
- --maxmemory
|
||||
- 64mb
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
- --port
|
||||
- '0'
|
||||
- --tls-port
|
||||
- '6379'
|
||||
- --tls-auth-clients
|
||||
- 'no'
|
||||
- --tls-cert-file
|
||||
- /run/tls/server/service.crt
|
||||
- --tls-key-file
|
||||
- /run/tls/server/service.key
|
||||
- --tls-ca-cert-file
|
||||
- /run/tls/trust/ca.crt
|
||||
- --aclfile
|
||||
- /run/secrets/cache.acl
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- valkey-cli --tls --cacert /run/tls/trust/ca.crt -h valkey ping 2>&1 | grep -q NOAUTH
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks:
|
||||
- backend
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/valkey:/run/tls/server:ro
|
||||
- ./secrets/cache.acl:/run/secrets/cache.acl:ro
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
build: &id001
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
user: 10001:10001
|
||||
read_only: true
|
||||
tmpfs: &id002
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop: &id003
|
||||
- ALL
|
||||
security_opt: &id004
|
||||
- no-new-privileges:true
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets:
|
||||
- postgres_password
|
||||
- db_api_password
|
||||
- db_worker_password
|
||||
- db_relay_password
|
||||
- db_reconciler_password
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab7
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4319
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
TLS_CA_FILE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/client/service.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/client/service.key
|
||||
networks: &id005
|
||||
- backend
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.db_roles
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
api:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
build: *id001
|
||||
user: 10001:10001
|
||||
read_only: true
|
||||
tmpfs: *id002
|
||||
cap_drop: *id003
|
||||
security_opt: *id004
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets: []
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab7
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4319
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
TLS_CA_FILE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/client/service.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/client/service.key
|
||||
BAO_ROLE: api
|
||||
ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456}
|
||||
ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2}
|
||||
ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1}
|
||||
networks: *id005
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
bucket-init:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
broker-tls:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/api-client:/run/tls/client:ro
|
||||
- ./secrets/tls/api:/run/tls/server:ro
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- import socket; socket.create_connection(('127.0.0.1',8080),1).close()
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 25
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/lb:/run/tls/server:ro
|
||||
- ./secrets/tls/lb-client:/run/tls/client:ro
|
||||
ports:
|
||||
- 127.0.0.1:${PUBLIC_PORT:-8443}:8443
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=4m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- backend
|
||||
broker:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command:
|
||||
- valkey-server
|
||||
- --appendonly
|
||||
- 'yes'
|
||||
- --appendfsync
|
||||
- always
|
||||
- --save
|
||||
- 60 1
|
||||
- --maxmemory
|
||||
- 192mb
|
||||
- --maxmemory-policy
|
||||
- noeviction
|
||||
- --bind
|
||||
- 127.0.0.1
|
||||
- --aclfile
|
||||
- /run/secrets/broker.acl
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
- ./secrets/broker.acl:/run/secrets/broker.acl:ro
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
memswap_limit: 384m
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- valkey-cli ping 2>&1 | grep -q NOAUTH
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks:
|
||||
- backend
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
relay:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
build: *id001
|
||||
user: 10001:10001
|
||||
read_only: true
|
||||
tmpfs: *id002
|
||||
cap_drop: *id003
|
||||
security_opt: *id004
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets: []
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab7
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4319
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
TLS_CA_FILE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/client/service.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/client/service.key
|
||||
BAO_ROLE: relay
|
||||
networks: *id005
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.relay
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
openbao:
|
||||
condition: service_started
|
||||
broker-tls:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/relay-client:/run/tls/client:ro
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
build: *id001
|
||||
user: 10001:10001
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m
|
||||
cap_drop: *id003
|
||||
security_opt: *id004
|
||||
pids_limit: 96
|
||||
cpus: 2
|
||||
mem_limit: 1536m
|
||||
memswap_limit: 1536m
|
||||
stop_grace_period: 7s
|
||||
secrets: []
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab7
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4319
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
TLS_CA_FILE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/client/service.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/client/service.key
|
||||
BAO_ROLE: worker
|
||||
networks: *id005
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.worker
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
bucket-init:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
broker-tls:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/worker-client:/run/tls/client:ro
|
||||
s3:
|
||||
image: mtusi-lab7-s3:2025-10-15
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/s3.Dockerfile
|
||||
target: server
|
||||
command:
|
||||
- server
|
||||
- /data
|
||||
- --address
|
||||
- :9000
|
||||
- --console-address
|
||||
- 127.0.0.1:9001
|
||||
- --certs-dir
|
||||
- /run/s3-tls
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./secrets/tls/s3/service.crt:/run/s3-tls/public.crt:ro
|
||||
- ./secrets/tls/s3/service.key:/run/s3-tls/private.key:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop:
|
||||
- ALL
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- wget
|
||||
- -q
|
||||
- -O
|
||||
- /dev/null
|
||||
- https://s3:9000/minio/health/live
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks:
|
||||
- backend
|
||||
environment:
|
||||
SSL_CERT_FILE: /run/tls/trust/ca.crt
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
bucket-init:
|
||||
image: mtusi-lab7-mc:2025-08-13
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/s3.Dockerfile
|
||||
target: client
|
||||
command:
|
||||
- /bootstrap-s3.sh
|
||||
volumes:
|
||||
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
|
||||
- ./infra/api-policy.json:/policies/api-policy.json:ro
|
||||
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
|
||||
- ./infra/worker-policy.json:/policies/worker-policy.json:ro
|
||||
- ./infra/test-policy.json:/policies/test-policy.json:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_reconciler_access_key_id
|
||||
- s3_reconciler_secret_access_key
|
||||
- s3_test_access_key_id
|
||||
- s3_test_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=16m
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
reconciler:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab7}
|
||||
build: *id001
|
||||
user: 10001:10001
|
||||
read_only: true
|
||||
tmpfs: *id002
|
||||
cap_drop: *id003
|
||||
security_opt: *id004
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets: []
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
APP_ENV: lab7
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
OTEL_SAMPLE_RATIO: ${OTEL_SAMPLE_RATIO:-1}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4319
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
TLS_CA_FILE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/tls/trust/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/tls/client/service.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/tls/client/service.key
|
||||
BAO_ROLE: reconciler
|
||||
networks: *id005
|
||||
profiles:
|
||||
- tools
|
||||
command:
|
||||
- python
|
||||
- -m
|
||||
- messenger.cleanup
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
bucket-init:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/reconciler-client:/run/tls/client:ro
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.147.0
|
||||
command:
|
||||
- --config=/etc/otelcol/config.yaml
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/collector-client:/run/tls/client:ro
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
networks:
|
||||
- backend
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.9.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=3d
|
||||
- --storage.tsdb.retention.size=256MB
|
||||
- --web.listen-address=127.0.0.1:19090
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/prometheus-client:/run/tls/client:ro
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
- backend
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
loki:
|
||||
image: grafana/loki:3.6.3
|
||||
command:
|
||||
- -config.file=/etc/loki/local.yaml
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro
|
||||
- loki_data:/loki
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
- backend
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
tempo:
|
||||
image: grafana/tempo:2.10.0
|
||||
command:
|
||||
- -config.file=/etc/tempo/local.yaml
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/local.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
mem_limit: 512m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
- backend
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
grafana:
|
||||
image: grafana/grafana:12.3.3
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: 'false'
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/tls/server/service.crt
|
||||
GF_SERVER_CERT_KEY: /run/tls/server/service.key
|
||||
GF_SECURITY_COOKIE_SECURE: 'true'
|
||||
secrets:
|
||||
- grafana_password
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- ./infra/otel/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
|
||||
- ./secrets/grafana-datasources:/etc/grafana/provisioning/datasources:ro
|
||||
- ./secrets/tls/grafana:/run/tls/server:ro
|
||||
ports:
|
||||
- 127.0.0.1:${GRAFANA_PORT:-3007}:3000
|
||||
mem_limit: 384m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
- backend
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
openbao:
|
||||
image: openbao/openbao:2.7.2
|
||||
user: 100:1000
|
||||
entrypoint:
|
||||
- /usr/bin/bao
|
||||
command:
|
||||
- server
|
||||
- -config=/openbao/config/server.hcl
|
||||
volumes:
|
||||
- ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro
|
||||
- bao_raft_state:/openbao/file
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/openbao:/run/tls/server:ro
|
||||
networks:
|
||||
- backend
|
||||
ports:
|
||||
- 127.0.0.1:${BAO_HOST_PORT:-8207}:8200
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=8m
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
broker-tls:
|
||||
image: haproxy:3.2.26-alpine
|
||||
network_mode: service:broker
|
||||
user: 10001:10001
|
||||
command:
|
||||
- haproxy
|
||||
- -W
|
||||
- -db
|
||||
- -f
|
||||
- /proxy/base.cfg
|
||||
- -f
|
||||
- /proxy/service.cfg
|
||||
volumes:
|
||||
- ./infra/security/proxy-base.cfg:/proxy/base.cfg:ro
|
||||
- ./infra/security/broker.cfg:/proxy/service.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/broker:/run/tls/server:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
depends_on:
|
||||
broker:
|
||||
condition: service_started
|
||||
restart: true
|
||||
otel-collector-tls:
|
||||
image: haproxy:3.2.26-alpine
|
||||
network_mode: service:otel-collector
|
||||
user: 10001:10001
|
||||
command:
|
||||
- haproxy
|
||||
- -W
|
||||
- -db
|
||||
- -f
|
||||
- /proxy/base.cfg
|
||||
- -f
|
||||
- /proxy/service.cfg
|
||||
volumes:
|
||||
- ./infra/security/proxy-base.cfg:/proxy/base.cfg:ro
|
||||
- ./infra/security/otel-collector.cfg:/proxy/service.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/otel-collector:/run/tls/server:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
depends_on:
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
restart: true
|
||||
loki-tls:
|
||||
image: haproxy:3.2.26-alpine
|
||||
network_mode: service:loki
|
||||
user: 10001:10001
|
||||
command:
|
||||
- haproxy
|
||||
- -W
|
||||
- -db
|
||||
- -f
|
||||
- /proxy/base.cfg
|
||||
- -f
|
||||
- /proxy/service.cfg
|
||||
volumes:
|
||||
- ./infra/security/proxy-base.cfg:/proxy/base.cfg:ro
|
||||
- ./infra/security/loki.cfg:/proxy/service.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/loki:/run/tls/server:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_started
|
||||
restart: true
|
||||
tempo-tls:
|
||||
image: haproxy:3.2.26-alpine
|
||||
network_mode: service:tempo
|
||||
user: 10001:10001
|
||||
command:
|
||||
- haproxy
|
||||
- -W
|
||||
- -db
|
||||
- -f
|
||||
- /proxy/base.cfg
|
||||
- -f
|
||||
- /proxy/service.cfg
|
||||
volumes:
|
||||
- ./infra/security/proxy-base.cfg:/proxy/base.cfg:ro
|
||||
- ./infra/security/tempo.cfg:/proxy/service.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/tempo:/run/tls/server:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
depends_on:
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: true
|
||||
prometheus-tls:
|
||||
image: haproxy:3.2.26-alpine
|
||||
network_mode: service:prometheus
|
||||
user: 10001:10001
|
||||
command:
|
||||
- haproxy
|
||||
- -W
|
||||
- -db
|
||||
- -f
|
||||
- /proxy/base.cfg
|
||||
- -f
|
||||
- /proxy/service.cfg
|
||||
volumes:
|
||||
- ./infra/security/proxy-base.cfg:/proxy/base.cfg:ro
|
||||
- ./infra/security/prometheus.cfg:/proxy/service.cfg:ro
|
||||
- ./secrets/tls/trust:/run/tls/trust:ro
|
||||
- ./secrets/tls/prometheus:/run/tls/server:ro
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 64m
|
||||
cpus: 0.25
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_started
|
||||
restart: true
|
||||
networks:
|
||||
backend: {}
|
||||
volumes:
|
||||
prometheus_data: null
|
||||
loki_data: null
|
||||
tempo_data: null
|
||||
grafana_data: null
|
||||
broker_data: null
|
||||
s3_data: null
|
||||
postgres_data: null
|
||||
bao_raft_state: null
|
||||
secrets:
|
||||
grafana_password:
|
||||
file: ./secrets/grafana_password
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
s3_test_access_key_id:
|
||||
file: ./secrets/s3_test_access_key_id
|
||||
s3_test_secret_access_key:
|
||||
file: ./secrets/s3_test_secret_access_key
|
||||
s3_worker_access_key_id:
|
||||
file: ./secrets/s3_worker_access_key_id
|
||||
s3_worker_secret_access_key:
|
||||
file: ./secrets/s3_worker_secret_access_key
|
||||
db_api_password:
|
||||
file: ./secrets/db_api_password
|
||||
db_worker_password:
|
||||
file: ./secrets/db_worker_password
|
||||
db_relay_password:
|
||||
file: ./secrets/db_relay_password
|
||||
db_reconciler_password:
|
||||
file: ./secrets/db_reconciler_password
|
||||
+19
-33
@@ -1,37 +1,23 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия ЛР7
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Публичный сертификат учебного CA находится в `secrets/tls/trust/ca.crt` после `make bootstrap`. У серверов отдельные DNS/IP SAN, срок leaf 7 суток. У клиентов URI `urn:mtusi:lab7:<identity>`. Все тесты используют проверку имени сервера. На HAProxy принимающая сторона проверяет цепочку, CRL и точный CN из списка разрешённых ролей. API проверяет URI SAN `lb-client` и CRL после проверки цепочки. CN и URI выпускает оператор закрытого CA.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Связь | Канал и допуск | Проверка |
|
||||
| --- | --- | --- |
|
||||
| Клиент → ingress | `https://localhost:8443`. CA и SAN `localhost` / `127.0.0.1`. Пользовательские операции проверяют Bearer. | `make test`, `make test-security`, чужой CA, ACL, logout/expiry и 429. |
|
||||
| LB → API | `api:8080`, mTLS, LB проверяет SAN `api`, API допускает URI `lb-client`. Проверки readiness используют тот же канал. | `scripts/security_probe.py` проверяет разрешённый leaf, отсутствие сертификата, чужой CA и сертификат не той роли от своего CA. |
|
||||
| API/worker/relay/reconciler → PostgreSQL | `postgres:5432`, TLS `verify-full`, отдельные SCRAM роли и ограниченные табличные права. Администратор доступен только операторскому migrate. | Worker читает задачи, получает отказ на users/messages/sessions и смену роли. `pg_stat_ssl` проверяет текущую сессию. |
|
||||
| API/worker/reconciler → S3 | `https://s3:9000`, отдельные ключи S3 и политики по prefix. Root секрет смонтирован только в S3 и операторский bucket-init. | Worker пишет в objects; запись в staging, удаление объекта и чтение политики bucket запрещены. |
|
||||
| API/relay/worker → broker | `broker:6380`, mTLS, CN api-client, relay-client или worker-client и отдельная Valkey ACL. API identity имеет только ping, relay публикует, worker потребляет. | Три отрицательных TLS случая, отсутствие сети на 6379, запрет команд и чужих ключей worker. |
|
||||
| API/relay/worker → Collector | `otel-collector:4319`, mTLS, разрешены их три service identity, только POST на OTLP. | Доступ корректного сертификата, три отрицательных TLS случая и проверка отсутствия обхода на 4318. |
|
||||
| Collector → Loki и Tempo | `loki:3100/otlp` и `tempo:4318`, mTLS с `collector-client`; разрешены только записи сигналов. | Логи и трассы доступны через Grafana, правила порта в `infra/security/`. |
|
||||
| Prometheus → Collector | TLS/mTLS `otel-collector:8889` и `:8888`; клиент `prometheus-client`, путь `/metrics`. | В Prometheus есть метрики мессенджера и счётчики экспорта; через Grafana проверяется запрос к источнику. |
|
||||
| Grafana → Prometheus/Loki/Tempo | TLS/mTLS с `grafana-client` на 9090/3100/3200. Политика допускает чтение. Ключ провижининга создаёт оператор локально. | Успех своей роли и отказ чужой для каждого источника в `scripts/security_probe.py`. |
|
||||
| API/worker/relay/reconciler → OpenBao | `https://openbao:8200` с CA и SAN. Cert auth требует leaf и URI роли; выдаёт токен на 90 с, max TTL 120 с, только свой KV path. | Чтение своего пути, отказ на чужой и на запись. Отозванный токен и старый leaf не принимаются. |
|
||||
| Оператор → OpenBao/Grafana/S3 | OpenBao `https://localhost:8207` с операторским root для bootstrap; Grafana `https://localhost:3007` с локальным паролем. S3 admin через операторский контейнер bucket-init по TLS. | Порты loopback. В приложениях отсутствуют root token, unseal share, S3 root и CA key. |
|
||||
| API → cache | `rediss://valkey:6379` с CA и hostname, отдельный пароль и ACL для `session-user:*`. Сессия проверяется в PostgreSQL. | Cache использует только TLS listener. `secrets/cache.acl` генерируется оператором, файл доступен контейнеру cache. |
|
||||
| Репликация хранилищ | N/A, один экземпляр каждого хранилища на лабораторном стенде. | Failover относится к API. |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
Plaintext после TLS-прокси ограничен loopback общего namespace контейнера назначения. В брокере это 127.0.0.1:6379, Collector 4318/18888/18889, Loki 13100, Tempo 13200/14318, Prometheus 19090. Внутренний gRPC Loki/Tempo также слушает loopback. API собственного plaintext listener не имеет; контейнерный healthcheck проверяет только открытие локального TLS сокета, а HAProxy проверяет прикладной readiness через mTLS. Unix-сокет PostgreSQL доступен самому контейнеру для bootstrap и health. Ни один служебный plaintext-порт не опубликован.
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
CA private key и unseal share в отдельной папке оператора под `secrets/operator/`. Она не монтируется в контейнеры. Контейнер получает только свои leaf-файлы и публичный trust. Процессы повторяют cert auth каждые 5 секунд, краткий токен отзывают сразу после чтения. После 30 секунд без подтверждения API перестаёт принимать бизнес-запросы, relay и worker не берут новые задачи. Начатая обработка ограничена timeout супервизора. При изменении KV сервис с прежними подключениями ждёт пересоздания оператором. Команды ротации и восстановления описаны в отчёте.
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 3, "first_recovery_seconds": 1.981}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1 @@
|
||||
wrong_hostname_rejected
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/security.py --base-url "https://localhost:8443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,411 @@
|
||||
{
|
||||
"scanner": "Trivy 0.69.3",
|
||||
"db_updated_utc": "2026-11-06T12:55:02.577403897Z",
|
||||
"scans": {
|
||||
"uv.lock": {
|
||||
"before": {
|
||||
"CRITICAL": 1
|
||||
},
|
||||
"after": {},
|
||||
"final_image_id": null,
|
||||
"high_critical": []
|
||||
},
|
||||
"Python app": {
|
||||
"before": {
|
||||
"HIGH": 54,
|
||||
"LOW": 100,
|
||||
"MEDIUM": 128,
|
||||
"UNKNOWN": 4,
|
||||
"CRITICAL": 7
|
||||
},
|
||||
"after": {
|
||||
"HIGH": 49,
|
||||
"LOW": 99,
|
||||
"MEDIUM": 126,
|
||||
"UNKNOWN": 4,
|
||||
"CRITICAL": 5
|
||||
},
|
||||
"final_image_id": "sha256:9af315bcff30e123aa6f89ec3dfc88190bf27d438b42b34cf65b3ead474f20c3",
|
||||
"high_critical": [
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-109518",
|
||||
"PkgName": "apt",
|
||||
"InstalledVersion": "2.6.1",
|
||||
"FixedVersion": "2.6.2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "bsdutils",
|
||||
"InstalledVersion": "1:2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "bsdutils",
|
||||
"InstalledVersion": "1:2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "bsdutils",
|
||||
"InstalledVersion": "1:2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-41992",
|
||||
"PkgName": "gzip",
|
||||
"InstalledVersion": "1.12-1",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-54369",
|
||||
"PkgName": "libacl1",
|
||||
"InstalledVersion": "2.3.1-3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-109518",
|
||||
"PkgName": "libapt-pkg6.0",
|
||||
"InstalledVersion": "2.6.1",
|
||||
"FixedVersion": "2.6.2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "libblkid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "libblkid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "libblkid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "libmount1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "libmount1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "libmount1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2025-69720",
|
||||
"PkgName": "libncursesw6",
|
||||
"InstalledVersion": "6.4-4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-103111",
|
||||
"PkgName": "libpcre2-8-0",
|
||||
"InstalledVersion": "10.42-1+deb12u1",
|
||||
"FixedVersion": "10.42-1+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "libsmartcols1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "libsmartcols1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "libsmartcols1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2025-7458",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "CRITICAL"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-11822",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-11824",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-119373",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-119374",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-119379",
|
||||
"PkgName": "libsqlite3-0",
|
||||
"InstalledVersion": "3.40.1-2+deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-84782",
|
||||
"PkgName": "libssl3",
|
||||
"InstalledVersion": "3.0.20-1~deb12u2",
|
||||
"FixedVersion": "3.0.22-1~deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-16742",
|
||||
"PkgName": "libsystemd0",
|
||||
"InstalledVersion": "252.39-1~deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2025-69720",
|
||||
"PkgName": "libtinfo6",
|
||||
"InstalledVersion": "6.4-4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-16742",
|
||||
"PkgName": "libudev1",
|
||||
"InstalledVersion": "252.39-1~deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "libuuid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "libuuid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "libuuid1",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "mount",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "mount",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "mount",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2025-69720",
|
||||
"PkgName": "ncurses-base",
|
||||
"InstalledVersion": "6.4-4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2025-69720",
|
||||
"PkgName": "ncurses-bin",
|
||||
"InstalledVersion": "6.4-4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-84782",
|
||||
"PkgName": "openssl",
|
||||
"InstalledVersion": "3.0.20-1~deb12u2",
|
||||
"FixedVersion": "3.0.22-1~deb12u2",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-13221",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "CRITICAL"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-42496",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "CRITICAL"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-8376",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "CRITICAL"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-42497",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-48962",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-57432",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-57433",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"FixedVersion": "5.36.0-7+deb12u4",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-9538",
|
||||
"PkgName": "perl-base",
|
||||
"InstalledVersion": "5.36.0-7+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "util-linux",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "util-linux",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "util-linux",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53613",
|
||||
"PkgName": "util-linux-extra",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-53615",
|
||||
"PkgName": "util-linux-extra",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-78410",
|
||||
"PkgName": "util-linux-extra",
|
||||
"InstalledVersion": "2.38.1-5+deb12u3",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2023-45853",
|
||||
"PkgName": "zlib1g",
|
||||
"InstalledVersion": "1:1.2.13.dfsg-1",
|
||||
"Severity": "CRITICAL"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-76844",
|
||||
"PkgName": "zlib1g",
|
||||
"InstalledVersion": "1:1.2.13.dfsg-1",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-85091",
|
||||
"PkgName": "zlib1g",
|
||||
"InstalledVersion": "1:1.2.13.dfsg-1",
|
||||
"Severity": "HIGH"
|
||||
}
|
||||
]
|
||||
},
|
||||
"HAProxy": {
|
||||
"before": {
|
||||
"MEDIUM": 37,
|
||||
"LOW": 54,
|
||||
"CRITICAL": 3,
|
||||
"HIGH": 24,
|
||||
"UNKNOWN": 2
|
||||
},
|
||||
"after": {},
|
||||
"final_image_id": "sha256:73b2c8d356752977b9dca994aced99ba706adfe299ccfc898b4c7268e10e58c0",
|
||||
"high_critical": []
|
||||
},
|
||||
"OpenBao": {
|
||||
"before": {
|
||||
"HIGH": 80,
|
||||
"CRITICAL": 8,
|
||||
"MEDIUM": 67,
|
||||
"LOW": 45,
|
||||
"UNKNOWN": 6
|
||||
},
|
||||
"after": {
|
||||
"HIGH": 2,
|
||||
"MEDIUM": 4,
|
||||
"UNKNOWN": 3
|
||||
},
|
||||
"final_image_id": "sha256:b66c2b796a7b05ae75dbf99b61c9adb804ea0b454f4abcb465a3da8b653f6357",
|
||||
"high_critical": [
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-116587",
|
||||
"PkgName": "github.com/go-jose/go-jose/v3",
|
||||
"InstalledVersion": "v3.0.5",
|
||||
"Severity": "HIGH"
|
||||
},
|
||||
{
|
||||
"VulnerabilityID": "CVE-2026-116587",
|
||||
"PkgName": "github.com/go-jose/go-jose/v4",
|
||||
"InstalledVersion": "v4.1.4",
|
||||
"FixedVersion": "4.1.6",
|
||||
"Severity": "HIGH"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# Сканирование зависимостей и образов
|
||||
|
||||
Trivy 0.69.3, база уязвимостей от 06.11.2026 12:55 UTC. Область сканирования: Python lockfile, runtime приложения, HAProxy и OpenBao. Архивы образов передавались сканеру без монтирования Docker socket. Результаты от сканера зависят от версии базы и состава платформы ARM64.
|
||||
|
||||
| Цель | HIGH/CRITICAL до | HIGH/CRITICAL после |
|
||||
| --- | ---: | ---: |
|
||||
| uv.lock | 0/1 | 0/0 |
|
||||
| Python app | 54/7 | 49/5 |
|
||||
| HAProxy | 24/3 | 0/0 |
|
||||
| OpenBao | 80/8 | 2/0 |
|
||||
|
||||
## Применимость и действия
|
||||
|
||||
- **GHSA-6q48-6c7w-9gw2, certifi.** В старом CA bundle находилась цепочка с компрометированными промежуточными ключами. Версия зависимости обновлена до 2026.10.23. Удалены отдельные pip и ensurepip базового runtime, где сканер нашёл ещё один старый встроенный bundle. Стенд проверяет собственный CA на внутренних соединениях. После сборки сканер не повторил это предупреждение.
|
||||
|
||||
- **HAProxy и libssl.** Старый Alpine image содержал уязвимые библиотеки обработки TLS и регулярных выражений. HAProxy обновлён до 3.2.26-alpine. Повторены HTTPS smoke и матрица mTLS на итоговой версии. [Официальные образы HAProxy](https://hub.docker.com/_/haproxy).
|
||||
|
||||
- **OpenBao.** Поднят 2.7.2 с Raft. Релиз закрывает в том числе GHSA-736f-82w7-hj22 в cert auth. Оставшиеся HIGH относятся к парсингу RSA JWK в go-jose (CVE-2026-116587). JWT/OIDC auth и импорт JWK в лабораторной конфигурации не включены; используется auth/cert. [Заметки релиза](https://github.com/openbao/openbao/releases/tag/v2.7.2).
|
||||
|
||||
- **Базовая ОС приложения.** Среди оставшихся находок CVE-2025-7458 (SQLite) и CVE-2023-45853 (MiniZip). API использует PostgreSQL и обработку PNG/JPEG, не принимает SQLite базы и ZIP. CVE в Perl относятся к выполнению Perl или распаковке архивов; на request path Perl не вызывается. Образ остаётся Debian-based и содержит системные пакеты с известными находками, в том числе в libssl. Для эксплуатации вне стенда нужна отдельная проверка актуальных образов, сетевой поверхности и обновлений Debian. [Debian Security Tracker](https://security-tracker.debian.org/tracker/).
|
||||
|
||||
Список HIGH/CRITICAL с установленными и исправленными версиями — [scan.json](scan.json). Отсутствие находок у отдельного образа отражает этот запуск Trivy.
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"tls": {
|
||||
"api": [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied"
|
||||
],
|
||||
"collector": [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied"
|
||||
],
|
||||
"broker": [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied"
|
||||
],
|
||||
"network_plaintext_bypass": "denied",
|
||||
"grafana_backends": {
|
||||
"prometheus": "verified_tls_identity_allow_and_deny",
|
||||
"loki": "verified_tls_identity_allow_and_deny",
|
||||
"tempo": "verified_tls_identity_allow_and_deny"
|
||||
}
|
||||
},
|
||||
"worker_scope": {
|
||||
"postgres": {
|
||||
"tls": true,
|
||||
"job_read": true,
|
||||
"denied_operations": 6
|
||||
},
|
||||
"broker": {
|
||||
"tls": true,
|
||||
"ping": true,
|
||||
"denied_operations": 3
|
||||
},
|
||||
"s3": {
|
||||
"put_objects": true,
|
||||
"denied_operations": 3
|
||||
},
|
||||
"openbao": {
|
||||
"own_read": true,
|
||||
"foreign_read_and_write_denied": true,
|
||||
"token_ttl_s": 90,
|
||||
"revoked_token_denied": true
|
||||
}
|
||||
},
|
||||
"api": {
|
||||
"statuses": [
|
||||
401,
|
||||
401,
|
||||
401,
|
||||
401,
|
||||
429,
|
||||
429
|
||||
],
|
||||
"replicas": 2,
|
||||
"latency_ms": [
|
||||
23.846624997531762,
|
||||
29.357125000387896,
|
||||
25.98266600034549,
|
||||
29.348417003348004,
|
||||
8.413042000029236,
|
||||
7.3904999990190845
|
||||
],
|
||||
"outsider_denied": true,
|
||||
"logout_revoked": true,
|
||||
"oversize_json_denied": true
|
||||
},
|
||||
"worker_compromise": {
|
||||
"old_identity": {
|
||||
"broker": "revoked",
|
||||
"collector": "revoked",
|
||||
"openbao": "revoked"
|
||||
},
|
||||
"accepted": 5,
|
||||
"ready_with_original": 5,
|
||||
"elapsed_seconds": 8.474,
|
||||
"broker_acl_reloaded": true,
|
||||
"images_rebuilt": false,
|
||||
"new_worker": {
|
||||
"postgres": {
|
||||
"tls": true,
|
||||
"job_read": true,
|
||||
"denied_operations": 6
|
||||
},
|
||||
"broker": {
|
||||
"tls": true,
|
||||
"ping": true,
|
||||
"denied_operations": 3
|
||||
},
|
||||
"s3": {
|
||||
"put_objects": true,
|
||||
"denied_operations": 3
|
||||
},
|
||||
"openbao": {
|
||||
"own_read": true,
|
||||
"foreign_read_and_write_denied": true,
|
||||
"token_ttl_s": 90,
|
||||
"revoked_token_denied": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"sealed": {
|
||||
"unready_after_seconds": 29.151,
|
||||
"recovered_after_seconds": 33.906
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: successful login rehashed old Argon2id parameters; expired session rejected.
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/trust/ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1 @@
|
||||
PASS: three Grafana datasources over mTLS; correlated payloads are clean.
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Never print mc arguments/errors: they can include access key identifiers.
|
||||
exec 3>&1
|
||||
exec > /tmp/bootstrap.log 2>&1
|
||||
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
|
||||
bucket="$S3_BUCKET"
|
||||
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
|
||||
mkdir -p /tmp/mc/certs/CAs
|
||||
cp /run/tls/trust/ca.crt /tmp/mc/certs/CAs/local.crt
|
||||
mc --config-dir /tmp/mc alias set -- local https://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
|
||||
for role in api worker reconciler test; do
|
||||
target_bucket="$bucket"
|
||||
policy_source=/policies/api-policy.json
|
||||
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
|
||||
if [ "$role" = worker ]; then policy_source=/policies/worker-policy.json; fi
|
||||
if [ "$role" = test ]; then
|
||||
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
|
||||
policy_source=/policies/test-policy.json
|
||||
target_bucket="$bucket-tests"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
|
||||
fi
|
||||
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
|
||||
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
|
||||
key="$(cat /run/secrets/s3_${role}_access_key_id)"
|
||||
mc --config-dir /tmp/mc admin user add -- local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
|
||||
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
|
||||
done
|
||||
trap - EXIT
|
||||
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
|
||||
@@ -0,0 +1,39 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2048
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout client 20s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 0s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8443 ssl crt /run/tls/server/service.pem
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
|
||||
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none ssl verify required ca-file /run/tls/trust/ca.crt verifyhost api sni str(api) check-sni api crt /run/tls/client/service.pem
|
||||
@@ -1,7 +1,7 @@
|
||||
# OpenBao — реализовать конфигурацию и политики
|
||||
# OpenBao
|
||||
|
||||
Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key.
|
||||
`make up` создаёт локальные secrets и PKI, запускает OpenBao в обычном режиме и вызывает `python3 scripts/bao_operator.py`. Скрипт инициализирует новый volume один раз и выполняет operator unseal после перезапуска. Операторский файл под `secrets/operator/` хранится на локальном защищённом диске и не монтируется в OpenBao. Его нужно сохранить отдельно от резервной копии volume. В учебном стенде один оператор и один share; при потере share восстановление содержимого требует отдельной копии.
|
||||
|
||||
Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения.
|
||||
Сервисы входят с собственным сертификатом через `auth/cert/login`, читают только `services/data/<роль>` и отзывают выданный токен. Проверка повторяется каждые пять секунд. Файлы `secrets/tls/` монтируются по роли. У OpenBao постоянный volume `bao_raft_state`; `docker compose restart openbao` переводит его в sealed, затем оператор запускает `python3 scripts/bao_operator.py`.
|
||||
|
||||
Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md.
|
||||
Скрипт проверки `scripts/security_probe.py --rotate --seal` выполняет описанный в отчёте сценарий отзыва и ротации на локальных синтетических данных. Использованные образы не пересобираются при смене сертификата или секрета.
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
ui = false
|
||||
api_addr = "https://openbao:8200"
|
||||
cluster_addr = "https://127.0.0.1:8201"
|
||||
storage "raft" {
|
||||
path = "/openbao/file"
|
||||
node_id = "lab7-local"
|
||||
performance_multiplier = 1
|
||||
}
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
cluster_address = "127.0.0.1:8201"
|
||||
tls_cert_file = "/run/tls/server/service.crt"
|
||||
tls_key_file = "/run/tls/server/service.key"
|
||||
tls_client_ca_file = "/run/tls/trust/ca.crt"
|
||||
tls_min_version = "tls12"
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: MessengerHttpErrors
|
||||
expr: |
|
||||
sum(rate(messenger_http_requests_total{route!~"/health/.*",status_class="5xx"}[20s]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[20s])), 0.01)
|
||||
> 0.05
|
||||
for: 6s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: More than 5% of business requests have failed for six seconds
|
||||
runbook: "REPORT.md#диагностика"
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 5 and max(messenger_queue_depth) > 0
|
||||
for: 6s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: The oldest unfinished job is more than five seconds old
|
||||
runbook: "REPORT.md#диагностика"
|
||||
- alert: MessengerDatabaseErrors
|
||||
expr: sum(increase(messenger_db_errors_total[20s])) > 0
|
||||
for: 4s
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: PostgreSQL requests are failing
|
||||
runbook: "REPORT.md#диагностика"
|
||||
@@ -0,0 +1,61 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 500ms
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 127.0.0.1:18889
|
||||
metric_expiration: 15s
|
||||
otlp_http/tempo:
|
||||
endpoint: https://tempo:4318
|
||||
tls:
|
||||
ca_file: /run/tls/trust/ca.crt
|
||||
cert_file: /run/tls/client/service.crt
|
||||
key_file: /run/tls/client/service.key
|
||||
timeout: 2s
|
||||
sending_queue: {queue_size: 16, num_consumers: 1}
|
||||
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
|
||||
otlp_http/loki:
|
||||
endpoint: https://loki:3100/otlp
|
||||
tls:
|
||||
ca_file: /run/tls/trust/ca.crt
|
||||
cert_file: /run/tls/client/service.crt
|
||||
key_file: /run/tls/client/service.key
|
||||
timeout: 2s
|
||||
sending_queue: {queue_size: 16, num_consumers: 1}
|
||||
retry_on_failure: {initial_interval: 500ms, max_interval: 2s, max_elapsed_time: 5s}
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 127.0.0.1
|
||||
port: 18888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
@@ -0,0 +1,453 @@
|
||||
{
|
||||
"uid": "messenger",
|
||||
"title": "MTUSI Messenger",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"editable": false,
|
||||
"timezone": "browser",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API / запросов в секунду",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API / доля 5xx",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[30s])),0.001)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API / p50, p95, p99",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[30s])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / исходы в секунду",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[30s]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / p95 по исходу",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le,outcome) (rate(messenger_jobs_duration_seconds_bucket[30s])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Upload → ready / p95",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_ready_duration_seconds_bucket[30s])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Backlog / заданий",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Возраст старейшего задания",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_seconds)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "PostgreSQL / занято и ожидает",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_db_pool_used",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "messenger_db_pool_waiting",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Worker / занято и доступно",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "CPU процессов",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(messenger_process_cpu_seconds_total[30s])",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "RSS процессов",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_rss_bytes",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Потери экспорта Collector",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_spans_total[30s]))",
|
||||
"legendFormat": ""
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (exporter) (increase(otelcol_exporter_send_failed_log_records_total[30s]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Алерты",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertname=~\"Messenger.*\"}",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "logs",
|
||||
"title": "API, relay и worker / связанные логи",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 24,
|
||||
"h": 10
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-(api|relay|worker)\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: MTUSI
|
||||
type: file
|
||||
allowUiUpdates: false
|
||||
options: {path: /var/lib/grafana/dashboards}
|
||||
@@ -0,0 +1,38 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
grpc_listen_address: 127.0.0.1
|
||||
http_listen_address: 127.0.0.1
|
||||
http_listen_port: 13100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index: {prefix: index_, period: 24h}
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
limits_config:
|
||||
retention_period: 72h
|
||||
allow_structured_metadata: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,23 @@
|
||||
global:
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 2s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger-otel
|
||||
honor_labels: true
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/tls/trust/ca.crt
|
||||
cert_file: /run/tls/client/service.crt
|
||||
key_file: /run/tls/client/service.key
|
||||
static_configs: [{targets: ["otel-collector:8889"]}]
|
||||
- job_name: collector
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/tls/trust/ca.crt
|
||||
cert_file: /run/tls/client/service.crt
|
||||
key_file: /run/tls/client/service.key
|
||||
static_configs: [{targets: ["otel-collector:8888"]}]
|
||||
@@ -0,0 +1,20 @@
|
||||
server:
|
||||
grpc_listen_address: 127.0.0.1
|
||||
http_listen_address: 127.0.0.1
|
||||
http_listen_port: 13200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:14318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal: {path: /var/tempo/wal}
|
||||
local: {path: /var/tempo/blocks}
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Official security-fixed source release; no dependency on retired image tags.
|
||||
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
|
||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
|
||||
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
|
||||
COPY --from=builder /go/bin/minio /usr/local/bin/minio
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
|
||||
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
|
||||
COPY --from=builder /go/bin/mc /usr/local/bin/mc
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bin/sh"]
|
||||
@@ -0,0 +1,7 @@
|
||||
listen broker
|
||||
mode tcp
|
||||
bind :6380 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str api-client relay-client worker-client
|
||||
tcp-request inspect-delay 1s
|
||||
tcp-request content reject unless allowed_identity
|
||||
server local 127.0.0.1:6379
|
||||
@@ -0,0 +1,13 @@
|
||||
listen loki
|
||||
mode http
|
||||
bind :3100 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str collector-client grafana-client
|
||||
http-request deny unless allowed_identity
|
||||
acl ingestion path /otlp/v1/logs
|
||||
acl collector ssl_c_s_dn(CN) -m str collector-client
|
||||
http-request deny if ingestion !collector
|
||||
http-request deny if collector !ingestion
|
||||
http-request deny if !collector !{ method GET POST }
|
||||
http-request deny if !collector !{ path_beg /loki/api/v1/ /ready }
|
||||
http-request deny if !collector { path_beg /loki/api/v1/delete }
|
||||
server local 127.0.0.1:13100
|
||||
@@ -0,0 +1,23 @@
|
||||
listen ingest
|
||||
mode http
|
||||
bind :4319 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str api-client worker-client relay-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { method POST } { path_beg /v1/ }
|
||||
server local 127.0.0.1:4318
|
||||
|
||||
listen scrape
|
||||
mode http
|
||||
bind :8889 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str prometheus-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { path /metrics }
|
||||
server local 127.0.0.1:18889
|
||||
|
||||
listen self_metrics
|
||||
mode http
|
||||
bind :8888 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str prometheus-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { path /metrics }
|
||||
server local 127.0.0.1:18888
|
||||
@@ -0,0 +1,3 @@
|
||||
local all all trust
|
||||
hostssl all all all scram-sha-256
|
||||
hostnossl all all all reject
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
install -d -m 0700 -o postgres /tmp/postgres-tls
|
||||
install -m 0600 -o postgres /run/tls/server/service.key /tmp/postgres-tls/server.key
|
||||
install -m 0644 -o postgres /run/tls/server/service.crt /tmp/postgres-tls/server.crt
|
||||
exec /usr/local/bin/docker-entrypoint.sh "$@"
|
||||
@@ -0,0 +1,8 @@
|
||||
listen prometheus
|
||||
mode http
|
||||
bind :9090 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str grafana-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { path_beg /api/v1/ /-/ready }
|
||||
http-request deny if { path_beg /api/v1/admin/ }
|
||||
server local 127.0.0.1:19090
|
||||
@@ -0,0 +1,11 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 256
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||
|
||||
defaults
|
||||
log global
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
retries 0
|
||||
@@ -0,0 +1,15 @@
|
||||
listen otlp
|
||||
mode http
|
||||
bind :4318 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str collector-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { method POST } { path /v1/traces }
|
||||
server local 127.0.0.1:14318
|
||||
|
||||
listen query
|
||||
mode http
|
||||
bind :3200 ssl crt /run/tls/server/service.pem verify required ca-file /run/tls/trust/ca.crt crl-file /run/tls/trust/revoked.crl
|
||||
acl allowed_identity ssl_c_s_dn(CN) -m str grafana-client
|
||||
http-request deny unless allowed_identity
|
||||
http-request deny unless { method GET } { path_beg /api/ /ready }
|
||||
server local 127.0.0.1:13200
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/staging/*",
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::BUCKET/objects/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id,user_id),
|
||||
UNIQUE (chat_id,position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE message_requests (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
idempotency_key varchar(128) COLLATE "C" NOT NULL
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
|
||||
response_message_id uuid NOT NULL REFERENCES messages(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (id,chat_id),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages ADD UNIQUE (id,chat_id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position),
|
||||
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
|
||||
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,46 @@
|
||||
-- Preserve earlier migration checksums; transition lab4 storage to versioned jobs.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_storage_state_check;
|
||||
ALTER TABLE attachments ADD CHECK (storage_state IN ('queued','processing','ready','failed')) NOT VALID;
|
||||
UPDATE attachments SET storage_state='queued' WHERE storage_state IN ('pending','missing');
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0);
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(variants)='array');
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
source_key text NOT NULL,
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued','processing','ready','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
fence uuid,
|
||||
lease_until timestamptz,
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
error_code text,
|
||||
source_expired boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (attachment_id,pipeline_version)
|
||||
);
|
||||
CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until,next_attempt_at);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
due_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
stream_id text
|
||||
);
|
||||
CREATE INDEX outbox_due ON job_outbox(due_at) WHERE published_at IS NULL;
|
||||
CREATE TABLE pipeline_audit (
|
||||
sequence bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
event text NOT NULL,
|
||||
details jsonb NOT NULL DEFAULT '{}',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Only an operator with DB access can configure faults; production ignores this table.
|
||||
CREATE TABLE pipeline_faults (
|
||||
job_id uuid REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
phase text NOT NULL,
|
||||
action text NOT NULL CHECK (action IN ('crash','hang','transient','permanent')),
|
||||
remaining integer NOT NULL DEFAULT 1 CHECK (remaining >= 0),
|
||||
PRIMARY KEY(job_id,phase)
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- A paused API must not acknowledge an object that GC has already retired.
|
||||
CREATE TABLE upload_reservations (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL DEFAULT 'uploading' CHECK (state IN ('uploading','accepted','abandoned')),
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Optional W3C context follows the durable job, including outbox retries.
|
||||
ALTER TABLE attachment_jobs ADD COLUMN traceparent text CHECK (
|
||||
traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'
|
||||
);
|
||||
ALTER TABLE attachment_jobs ADD COLUMN tracestate text CHECK (length(tracestate) <= 512);
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE login_budget (
|
||||
scope text NOT NULL,
|
||||
window_start bigint NOT NULL,
|
||||
attempts integer NOT NULL CHECK (attempts > 0),
|
||||
PRIMARY KEY (scope, window_start)
|
||||
);
|
||||
CREATE INDEX login_budget_window ON login_budget(window_start);
|
||||
@@ -0,0 +1,48 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "0.7.0"
|
||||
description = "MTUSI lab 7: messenger service security"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"opentelemetry-sdk==1.45.2",
|
||||
"opentelemetry-exporter-otlp-proto-http==1.45.2",
|
||||
"argon2-cffi==25.1.0",
|
||||
"boto3==1.43.109",
|
||||
"fastapi==0.143.3",
|
||||
"pillow==13.0.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"pydantic==2.14.1",
|
||||
"redis==8.1.0",
|
||||
"uvicorn==0.54.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling==1.27.0"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/messenger"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"httpx==0.28.1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"pytest==9.1.1",
|
||||
"ruff==0.17.1",
|
||||
]
|
||||
research = ["matplotlib==3.11.2"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra --tb=short"
|
||||
|
||||
[tool.ruff]
|
||||
extend-exclude = ["tests", "scripts/check_contract.py", "scripts/validate_openapi.py"]
|
||||
target-version = "py314"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator-only init, unseal, service policies and scoped secrets."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from bootstrap import ROLES, SECRETS
|
||||
|
||||
ADDRESS = f"https://localhost:{int(os.getenv('BAO_HOST_PORT', '8207'))}/v1/"
|
||||
CA = SECRETS / "tls/trust/ca.crt"
|
||||
RECEIPT = SECRETS / "operator/bao.json"
|
||||
|
||||
|
||||
def call(path, data=None, *, token=None, method=None):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
req = Request(
|
||||
ADDRESS + path,
|
||||
json.dumps(data).encode() if data is not None else None,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
with urlopen(req, context=ssl.create_default_context(cafile=CA), timeout=30) as response:
|
||||
body = response.read()
|
||||
return json.loads(body) if body else {}
|
||||
|
||||
|
||||
def secret(name):
|
||||
return (SECRETS / name).read_text().strip()
|
||||
|
||||
|
||||
def config(role):
|
||||
ca = "/run/tls/trust/ca.crt"
|
||||
db = (
|
||||
f"postgresql://{role}:{secret(f'db_{role}_password')}@postgres:5432/messenger"
|
||||
f"?sslmode=verify-full&sslrootcert={ca}"
|
||||
)
|
||||
cache = (
|
||||
f"rediss://{role}:{secret(f'cache_{role}_password')}@valkey:6379/0"
|
||||
f"?ssl_ca_certs={ca}&ssl_check_hostname=True"
|
||||
if role == "api"
|
||||
else ""
|
||||
)
|
||||
broker = (
|
||||
f"rediss://{role}:{secret(f'broker_{role}_password')}@broker:6380/0"
|
||||
f"?ssl_ca_certs={ca}&ssl_check_hostname=True&ssl_certfile=/run/tls/client/service.crt"
|
||||
"&ssl_keyfile=/run/tls/client/service.key"
|
||||
if role in ("api", "worker", "relay")
|
||||
else ""
|
||||
)
|
||||
return {
|
||||
"database_url": db,
|
||||
"cache_url": cache,
|
||||
"broker_url": broker,
|
||||
"s3_access_key": secret(f"s3_{role}_access_key_id") if role != "relay" else "",
|
||||
"s3_secret_key": secret(f"s3_{role}_secret_access_key") if role != "relay" else "",
|
||||
}
|
||||
|
||||
|
||||
def setup():
|
||||
os.umask(0o077)
|
||||
for attempt in range(20):
|
||||
try:
|
||||
state = call("sys/init")
|
||||
break
|
||||
except URLError:
|
||||
if attempt == 19:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
if not state["initialized"]:
|
||||
result = call("sys/init", {"secret_shares": 1, "secret_threshold": 1}, method="PUT")
|
||||
RECEIPT.write_text(
|
||||
json.dumps({"root_token": result["root_token"], "unseal_key": result["keys_base64"][0]})
|
||||
)
|
||||
if not RECEIPT.exists():
|
||||
raise SystemExit("This OpenBao volume requires its original operator receipt")
|
||||
receipt = json.loads(RECEIPT.read_text())
|
||||
if call("sys/seal-status")["sealed"]:
|
||||
call("sys/unseal", {"key": receipt["unseal_key"]}, method="PUT")
|
||||
for attempt in range(30):
|
||||
try:
|
||||
call("sys/health")
|
||||
break
|
||||
except HTTPError as exc:
|
||||
if exc.code not in (429, 503) or attempt == 29:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
token = receipt["root_token"]
|
||||
if "cert/" not in call("sys/auth", token=token):
|
||||
call("sys/auth/cert", {"type": "cert"}, token=token)
|
||||
if "services/" not in call("sys/mounts", token=token):
|
||||
call("sys/mounts/services", {"type": "kv", "options": {"version": "2"}}, token=token)
|
||||
for role in ROLES:
|
||||
policy = (
|
||||
f'path "services/data/{role}" {{ capabilities = ["read"] }}\n'
|
||||
'path "auth/token/revoke-self" { capabilities = ["update"] }\n'
|
||||
)
|
||||
call("sys/policies/acl/" + role, {"policy": policy}, token=token, method="PUT")
|
||||
cert = secret(f"tls/{role}-client/service.crt")
|
||||
call(
|
||||
f"auth/cert/certs/{role}",
|
||||
{
|
||||
"certificate": cert,
|
||||
"allowed_uri_sans": f"urn:mtusi:lab7:{role}-client",
|
||||
"token_policies": [role],
|
||||
"token_ttl": "90s",
|
||||
"token_max_ttl": "120s",
|
||||
"token_no_default_policy": True,
|
||||
},
|
||||
token=token,
|
||||
)
|
||||
call(f"services/data/{role}", {"data": config(role)}, token=token)
|
||||
print("OpenBao initialized and unsealed; four scoped identities configured.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--revoke", choices=ROLES)
|
||||
parser.add_argument("--seal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.revoke or args.seal:
|
||||
token = json.loads(RECEIPT.read_text())["root_token"]
|
||||
if args.revoke:
|
||||
call(f"auth/cert/certs/{args.revoke}", token=token, method="DELETE")
|
||||
print(f"Certificate role {args.revoke} revoked.")
|
||||
else:
|
||||
call("sys/seal", {}, token=token, method="PUT")
|
||||
print("OpenBao sealed.")
|
||||
else:
|
||||
setup()
|
||||
except HTTPError as exc:
|
||||
raise SystemExit(f"OpenBao operator action failed (HTTP {exc.code})") from None
|
||||
except (URLError, TimeoutError):
|
||||
raise SystemExit("OpenBao unavailable over verified TLS") from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local operator credentials and TLS files for this stand."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pki
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
ROLES = ("api", "worker", "relay", "reconciler")
|
||||
S3_ROLES = ("api", "worker", "reconciler", "test")
|
||||
|
||||
|
||||
def ensure(name, value):
|
||||
path = SECRETS / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
path.chmod(0o644)
|
||||
return path.read_text().strip()
|
||||
|
||||
|
||||
def initialize():
|
||||
os.umask(0o077)
|
||||
SECRETS.mkdir(mode=0o700, exist_ok=True)
|
||||
pki.init_ca()
|
||||
for name in (*pki.SERVERS, *pki.CLIENTS):
|
||||
pki.issue(name)
|
||||
env = ROOT / ".env"
|
||||
if not env.exists():
|
||||
shutil.copyfile(ROOT / ".env.example", env)
|
||||
for name in ("postgres_password", "grafana_password", "s3_root_password"):
|
||||
ensure(name, secrets.token_hex(24))
|
||||
ensure("s3_root_user", "operator-" + secrets.token_hex(8))
|
||||
for role in S3_ROLES:
|
||||
ensure(f"s3_{role}_access_key_id", "svc-" + secrets.token_hex(8))
|
||||
ensure(f"s3_{role}_secret_access_key", secrets.token_hex(24))
|
||||
for role in ROLES:
|
||||
for store in ("db", "cache", "broker"):
|
||||
ensure(f"{store}_{role}_password", secrets.token_hex(24))
|
||||
write_acls()
|
||||
provision_datasources()
|
||||
print("Local PKI, credentials and provisioning ready.")
|
||||
|
||||
|
||||
def write_acls():
|
||||
cache_ops = "reset on ~session-user:* +get +set +del +ping +hello +client|setinfo"
|
||||
broker_shared = "+ping +hello +client|setinfo"
|
||||
worker_ops = (
|
||||
"~attachment-jobs +xreadgroup +xautoclaim +xgroup|create "
|
||||
"+xack +xdel +eval +multi +exec (~invalid-jobs +xadd)"
|
||||
)
|
||||
for store, rules in {
|
||||
"cache": {"api": cache_ops, "operator": "on ~* +@all"},
|
||||
"broker": {
|
||||
"api": f"on {broker_shared}",
|
||||
"relay": f"on ~attachment-jobs +xadd {broker_shared}",
|
||||
"worker": f"on {worker_ops} {broker_shared}",
|
||||
"operator": "on ~* +@all",
|
||||
},
|
||||
}.items():
|
||||
lines = ["user default off"]
|
||||
for role, permissions in rules.items():
|
||||
password = ensure(f"{store}_{role}_password", secrets.token_hex(24))
|
||||
lines.append(f"user {role} {permissions} >{password}")
|
||||
target = SECRETS / f"{store}.acl"
|
||||
target.write_text("\n".join(lines) + "\n")
|
||||
target.chmod(0o644)
|
||||
|
||||
|
||||
def provision_datasources():
|
||||
trust = (pki.TLS / "trust" / "ca.crt").read_text()
|
||||
identity = pki.TLS / "grafana-client"
|
||||
secure = {
|
||||
"tlsCACert": trust,
|
||||
"tlsClientCert": (identity / "service.crt").read_text(),
|
||||
"tlsClientKey": (identity / "service.key").read_text(),
|
||||
}
|
||||
datasources = []
|
||||
for name, port in (("prometheus", 9090), ("loki", 3100), ("tempo", 3200)):
|
||||
config = {"tlsAuth": True, "tlsAuthWithCACert": True}
|
||||
if name == "loki":
|
||||
config["derivedFields"] = [
|
||||
{
|
||||
"name": "Trace",
|
||||
"matcherType": "label",
|
||||
"matcherRegex": "trace_id",
|
||||
"datasourceUid": "tempo",
|
||||
"url": "$${__value.raw}",
|
||||
}
|
||||
]
|
||||
if name == "tempo":
|
||||
config["tracesToLogsV2"] = {
|
||||
"datasourceUid": "loki",
|
||||
"spanStartTimeShift": "-2m",
|
||||
"spanEndTimeShift": "2m",
|
||||
"filterByTraceID": True,
|
||||
"tags": [{"key": "service.name", "value": "service_name"}],
|
||||
}
|
||||
datasources.append(
|
||||
{
|
||||
"name": name.title(),
|
||||
"uid": name,
|
||||
"type": name,
|
||||
"access": "proxy",
|
||||
"url": f"https://{name}:{port}",
|
||||
"editable": False,
|
||||
"isDefault": name == "prometheus",
|
||||
"jsonData": config,
|
||||
"secureJsonData": secure,
|
||||
}
|
||||
)
|
||||
directory = SECRETS / "grafana-datasources"
|
||||
directory.mkdir(exist_ok=True)
|
||||
directory.chmod(0o755)
|
||||
target = directory / "local.yaml"
|
||||
target.write_text(json.dumps({"apiVersion": 1, "datasources": datasources}, indent=2))
|
||||
target.chmod(0o644)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
initialize()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
|
||||
|
||||
import json
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="https://localhost:8443", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
tls = ssl.create_default_context(cafile=ROOT / "secrets/tls/trust/ca.crt")
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2, context=tls) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator PKI. Only service-specific files are mounted into containers."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / "secrets"
|
||||
CA = ROOT / "operator" / "ca"
|
||||
TLS = ROOT / "tls"
|
||||
SERVERS = {
|
||||
"lb": "DNS:localhost,DNS:lb,IP:127.0.0.1",
|
||||
"api": "DNS:api",
|
||||
"postgres": "DNS:postgres",
|
||||
"valkey": "DNS:valkey",
|
||||
"broker": "DNS:broker",
|
||||
"s3": "DNS:s3",
|
||||
"openbao": "DNS:openbao,DNS:localhost,IP:127.0.0.1",
|
||||
"otel-collector": "DNS:otel-collector",
|
||||
"prometheus": "DNS:prometheus",
|
||||
"loki": "DNS:loki",
|
||||
"tempo": "DNS:tempo",
|
||||
"grafana": "DNS:grafana,DNS:localhost,IP:127.0.0.1",
|
||||
}
|
||||
CLIENTS = (
|
||||
"lb-client",
|
||||
"api-client",
|
||||
"worker-client",
|
||||
"relay-client",
|
||||
"reconciler-client",
|
||||
"collector-client",
|
||||
"prometheus-client",
|
||||
"grafana-client",
|
||||
"probe-client",
|
||||
)
|
||||
|
||||
|
||||
def openssl(*args):
|
||||
subprocess.run(
|
||||
["openssl", *map(str, args)],
|
||||
cwd=CA,
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def init_ca():
|
||||
CA.mkdir(parents=True, exist_ok=True)
|
||||
TLS.mkdir(parents=True, exist_ok=True)
|
||||
if (CA / "ca.key").exists():
|
||||
return
|
||||
(CA / "issued").mkdir(exist_ok=True)
|
||||
for name, value in {
|
||||
"index.txt": "",
|
||||
"index.txt.attr": "unique_subject = no\n",
|
||||
"serial": "1000\n",
|
||||
"crlnumber": "1000\n",
|
||||
}.items():
|
||||
(CA / name).write_text(value)
|
||||
openssl(
|
||||
"req",
|
||||
"-new",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-x509",
|
||||
"-days",
|
||||
90,
|
||||
"-subj",
|
||||
"/CN=MTUSI lab7 local root",
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:TRUE,pathlen:0",
|
||||
"-addext",
|
||||
"keyUsage=critical,keyCertSign,cRLSign",
|
||||
"-keyout",
|
||||
"ca.key",
|
||||
"-out",
|
||||
"ca.crt",
|
||||
)
|
||||
(CA / "ca.cnf").write_text("""[ca]
|
||||
default_ca = signer
|
||||
[signer]
|
||||
database = index.txt
|
||||
new_certs_dir = issued
|
||||
certificate = ca.crt
|
||||
private_key = ca.key
|
||||
serial = serial
|
||||
crlnumber = crlnumber
|
||||
default_md = sha256
|
||||
default_days = 7
|
||||
default_crl_days = 30
|
||||
policy = policy
|
||||
unique_subject = no
|
||||
[policy]
|
||||
commonName = supplied
|
||||
""")
|
||||
refresh_crl()
|
||||
|
||||
|
||||
def refresh_crl():
|
||||
openssl("ca", "-config", "ca.cnf", "-gencrl", "-out", "current.crl")
|
||||
trust = TLS / "trust"
|
||||
trust.mkdir(exist_ok=True)
|
||||
trust.chmod(0o755)
|
||||
for source, name in (("ca.crt", "ca.crt"), ("current.crl", "revoked.crl")):
|
||||
data = (CA / source).read_bytes()
|
||||
(trust / name).write_bytes(data)
|
||||
(trust / name).chmod(0o644)
|
||||
|
||||
|
||||
def issue(name, *, rotate=False):
|
||||
target = TLS / name
|
||||
target.mkdir(exist_ok=True)
|
||||
target.chmod(0o755)
|
||||
key, cert = target / "service.key", target / "service.crt"
|
||||
if cert.exists() and not rotate:
|
||||
return
|
||||
if rotate and cert.exists():
|
||||
openssl("ca", "-config", "ca.cnf", "-revoke", cert)
|
||||
purpose = "serverAuth" if name in SERVERS else "clientAuth"
|
||||
san = SERVERS.get(name, "URI:urn:mtusi:lab7:" + name)
|
||||
(CA / "leaf.cnf").write_text(
|
||||
"[leaf]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n"
|
||||
f"extendedKeyUsage={purpose}\nsubjectAltName={san}\n"
|
||||
"subjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid,issuer\n"
|
||||
)
|
||||
openssl(
|
||||
"req",
|
||||
"-new",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-subj",
|
||||
"/CN=" + name,
|
||||
"-keyout",
|
||||
"leaf.key",
|
||||
"-out",
|
||||
"leaf.csr",
|
||||
)
|
||||
openssl(
|
||||
"ca",
|
||||
"-config",
|
||||
"ca.cnf",
|
||||
"-batch",
|
||||
"-notext",
|
||||
"-extfile",
|
||||
"leaf.cnf",
|
||||
"-extensions",
|
||||
"leaf",
|
||||
"-in",
|
||||
"leaf.csr",
|
||||
"-out",
|
||||
"leaf.crt",
|
||||
)
|
||||
# A directory mount sees atomic leaf replacement. Host parent remains 0700.
|
||||
for source, dest in (("leaf.key", key), ("leaf.crt", cert)):
|
||||
temp = dest.with_suffix(".new")
|
||||
shutil.move(CA / source, temp)
|
||||
temp.chmod(0o644)
|
||||
temp.replace(dest)
|
||||
(target / "service.pem").write_bytes(cert.read_bytes() + key.read_bytes())
|
||||
(target / "service.pem").chmod(0o644)
|
||||
if rotate:
|
||||
refresh_crl()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--rotate", choices=[*SERVERS, *CLIENTS])
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
ROOT.mkdir(exist_ok=True, mode=0o700)
|
||||
init_ca()
|
||||
if args.rotate:
|
||||
issue(args.rotate, rotate=True)
|
||||
else:
|
||||
for name in (*SERVERS, *CLIENTS):
|
||||
issue(name)
|
||||
print("Local PKI ready.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api worker relay lb
|
||||
docker compose restart --timeout 7 postgres valkey s3 broker openbao
|
||||
python3 scripts/bao_operator.py
|
||||
docker compose restart broker-tls
|
||||
docker compose up -d --no-build --wait --wait-timeout 90 api worker relay lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible negative checks and worker revocation on the local lab7 stand."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCAL = ROOT / "secrets"
|
||||
CA = LOCAL / "tls/trust/ca.crt"
|
||||
INSIDE_CA = "/run/tls/trust/ca.crt"
|
||||
|
||||
|
||||
def checked(*command, **kwargs):
|
||||
try:
|
||||
return subprocess.run(command, cwd=ROOT, check=True, **kwargs)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
if exc.stderr:
|
||||
sys.stderr.write(exc.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return checked("docker", "compose", *args, **kwargs)
|
||||
|
||||
|
||||
def context(ca, certificate=None):
|
||||
ctx = ssl.create_default_context(cafile=ca)
|
||||
if certificate:
|
||||
ctx.load_cert_chain(certificate / "service.crt", certificate / "service.key")
|
||||
return ctx
|
||||
|
||||
|
||||
def response(url, ctx, data=None, headers=None, method=None):
|
||||
request = Request(url, data, headers or {}, method=method)
|
||||
try:
|
||||
with urlopen(request, context=ctx, timeout=4) as r:
|
||||
return r.status, r.read(262144)
|
||||
except HTTPError as e:
|
||||
return e.code, b""
|
||||
|
||||
|
||||
def denied(action):
|
||||
try:
|
||||
status = action()
|
||||
assert status in (401, 403) or status == b"", "Unexpected success in negative probe"
|
||||
except (ssl.SSLError, ConnectionResetError, BrokenPipeError, URLError) as exc:
|
||||
# Preserve DNS/connectivity errors as real test failures.
|
||||
cause = exc.reason if isinstance(exc, URLError) else exc
|
||||
if not isinstance(cause, (ssl.SSLError, ConnectionResetError, BrokenPipeError)):
|
||||
raise
|
||||
|
||||
|
||||
def redis_ping(ctx):
|
||||
with socket.create_connection(("broker", 6380), timeout=3) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname="broker") as tls:
|
||||
tls.sendall(b"*1\r\n$4\r\nPING\r\n")
|
||||
return tls.recv(256)
|
||||
|
||||
|
||||
def audit_tls():
|
||||
results = {}
|
||||
endpoints = {
|
||||
"api": (lambda ctx: response("https://api:8080/health/ready", ctx)[0], "lb-client"),
|
||||
"collector": (
|
||||
lambda ctx: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0],
|
||||
"worker-client",
|
||||
),
|
||||
"broker": (redis_ping, "worker-client"),
|
||||
}
|
||||
for name, (probe, identity) in endpoints.items():
|
||||
status = probe(context(INSIDE_CA, Path("/probe") / identity))
|
||||
assert status == 200 or status.startswith(b"-NOAUTH")
|
||||
for cert in (None, Path("/probe/probe-client"), Path("/probe/rogue")):
|
||||
denied(lambda probe=probe, cert=cert: probe(context(INSIDE_CA, cert)))
|
||||
results[name] = [
|
||||
"allowed",
|
||||
"no_client_denied",
|
||||
"same_ca_wrong_role_denied",
|
||||
"rogue_ca_denied",
|
||||
]
|
||||
|
||||
for host, port in (
|
||||
("broker", 6379),
|
||||
("otel-collector", 4318),
|
||||
("prometheus", 19090),
|
||||
("loki", 13100),
|
||||
("tempo", 13200),
|
||||
("tempo", 14318),
|
||||
("tempo", 9095),
|
||||
("loki", 9095),
|
||||
):
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1):
|
||||
raise AssertionError("Reachable plaintext " + host)
|
||||
except (ConnectionRefusedError, TimeoutError):
|
||||
pass
|
||||
results["network_plaintext_bypass"] = "denied"
|
||||
results["grafana_backends"] = {}
|
||||
for host, port, path in (
|
||||
("prometheus", 9090, "/api/v1/status/buildinfo"),
|
||||
("loki", 3100, "/loki/api/v1/status/buildinfo"),
|
||||
("tempo", 3200, "/api/echo"),
|
||||
):
|
||||
endpoint = f"https://{host}:{port}{path}"
|
||||
assert response(endpoint, context(INSIDE_CA, Path("/probe/grafana-client")))[0] == 200
|
||||
denied(
|
||||
lambda endpoint=endpoint: response(
|
||||
endpoint, context(INSIDE_CA, Path("/probe/probe-client"))
|
||||
)[0]
|
||||
)
|
||||
results["grafana_backends"][host] = "verified_tls_identity_allow_and_deny"
|
||||
return results
|
||||
|
||||
|
||||
def worker_scope():
|
||||
import psycopg
|
||||
import redis
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
settings = Settings.from_env()
|
||||
with psycopg.connect(settings.database_url, autocommit=True) as conn:
|
||||
assert conn.execute("SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()").fetchone()[0]
|
||||
conn.execute("SELECT count(*) FROM attachment_jobs").fetchone()
|
||||
for query in (
|
||||
"SELECT count(*) FROM users",
|
||||
"SELECT count(*) FROM messages",
|
||||
"SELECT count(*) FROM sessions",
|
||||
"CREATE TEMP TABLE forbidden_probe(id int)",
|
||||
"UPDATE attachments SET owner_id=owner_id WHERE false",
|
||||
"SET ROLE messenger",
|
||||
):
|
||||
try:
|
||||
conn.execute(query)
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
continue
|
||||
raise AssertionError("Unexpected database grant")
|
||||
|
||||
broker = redis.Redis.from_url(settings.broker_url)
|
||||
assert broker.ping()
|
||||
for op in (
|
||||
lambda: broker.config_get("appendonly"),
|
||||
lambda: broker.xrange("unrelated-key"),
|
||||
lambda: broker.get("session-user:synthetic"),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except redis.exceptions.NoPermissionError:
|
||||
continue
|
||||
raise AssertionError("Unexpected broker grant")
|
||||
|
||||
store = ObjectStore(settings)
|
||||
suffix = uuid.uuid4().hex
|
||||
key = "objects/audit/" + suffix
|
||||
store.client.put_object(Bucket=store.bucket, Key=key, Body=b"synthetic")
|
||||
assert store.client.head_object(Bucket=store.bucket, Key=key)["ContentLength"] == 9
|
||||
for op in (
|
||||
lambda: store.client.put_object(
|
||||
Bucket=store.bucket, Key="staging/audit/" + suffix, Body=b"x"
|
||||
),
|
||||
lambda: store.client.delete_object(Bucket=store.bucket, Key=key),
|
||||
lambda: store.client.get_bucket_policy(Bucket=store.bucket),
|
||||
):
|
||||
try:
|
||||
op()
|
||||
except ClientError as exc:
|
||||
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
continue
|
||||
raise AssertionError("Unexpected S3 grant")
|
||||
|
||||
cert = Path("/run/tls/client")
|
||||
ctx = context(INSIDE_CA, cert)
|
||||
base = "https://openbao:8200/v1/"
|
||||
ct = {"Content-Type": "application/json"}
|
||||
code, raw = response(base + "auth/cert/login", ctx, b'{"name":"worker"}', ct)
|
||||
assert code == 200
|
||||
auth = json.loads(raw)["auth"]
|
||||
token = auth["client_token"]
|
||||
headers = {**ct, "X-Vault-Token": token}
|
||||
assert 0 < auth["lease_duration"] <= 120
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 200
|
||||
assert response(base + "services/data/api", ctx, headers=headers)[0] == 403
|
||||
assert response(base + "services/data/worker", ctx, b'{"data":{}}', headers)[0] == 403
|
||||
assert response(base + "auth/token/revoke-self", ctx, b"{}", headers)[0] == 204
|
||||
assert response(base + "services/data/worker", ctx, headers=headers)[0] == 403
|
||||
return {
|
||||
"postgres": {"tls": True, "job_read": True, "denied_operations": 6},
|
||||
"broker": {"tls": True, "ping": True, "denied_operations": 3},
|
||||
"s3": {"put_objects": True, "denied_operations": 3},
|
||||
"openbao": {
|
||||
"own_read": True,
|
||||
"foreign_read_and_write_denied": True,
|
||||
"token_ttl_s": auth["lease_duration"],
|
||||
"revoked_token_denied": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def old_identity():
|
||||
ctx = context(INSIDE_CA, Path("/probe/old"))
|
||||
denied(lambda: redis_ping(ctx))
|
||||
denied(
|
||||
lambda: response(
|
||||
"https://otel-collector:4319/v1/traces",
|
||||
ctx,
|
||||
b"",
|
||||
{"Content-Type": "application/x-protobuf"},
|
||||
)[0]
|
||||
)
|
||||
code, _ = response(
|
||||
"https://openbao:8200/v1/auth/cert/login",
|
||||
ctx,
|
||||
b'{"name":"worker"}',
|
||||
{"Content-Type": "application/json"},
|
||||
)
|
||||
assert code in (400, 403)
|
||||
return {"broker": "revoked", "collector": "revoked", "openbao": "revoked"}
|
||||
|
||||
|
||||
def worker_probe():
|
||||
raw = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"worker",
|
||||
"/test/audit.py",
|
||||
"--inside",
|
||||
"worker",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def isolated(mode, mounts):
|
||||
cmd = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
"mtusi-lab7_backend",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--security-opt",
|
||||
"no-new-privileges:true",
|
||||
"--memory",
|
||||
"256m",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/audit.py:ro",
|
||||
"-v",
|
||||
f"{LOCAL / 'tls/trust'}:/run/tls/trust:ro",
|
||||
]
|
||||
for name, source in mounts.items():
|
||||
cmd += ["-v", f"{source}:/probe/{name}:ro"]
|
||||
image = compose("images", "-q", "api", capture_output=True, text=True).stdout.split()[0]
|
||||
raw = checked(
|
||||
*cmd, image, "python", "/test/audit.py", "--inside", mode, capture_output=True, text=True
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def verify_hostname():
|
||||
with socket.create_connection(("127.0.0.1", 8443), timeout=3) as raw:
|
||||
try:
|
||||
with context(CA).wrap_socket(raw, server_hostname="wrong-name.invalid"):
|
||||
raise AssertionError("Wrong TLS hostname accepted")
|
||||
except ssl.SSLCertVerificationError as exc:
|
||||
assert exc.verify_code == 62, "Expected a hostname mismatch"
|
||||
return "wrong_hostname_rejected"
|
||||
|
||||
|
||||
def check_api(client):
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, attachment)
|
||||
path = "/api/v1/attachments/" + attachment["id"]
|
||||
client.call("GET", path, 404, actor=eve, headers={"X-User-Id": alice["id"]})
|
||||
client.call("GET", path + "/content", 404, actor=eve)
|
||||
client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=bob)
|
||||
reg = alice["registration"]
|
||||
bad = base64.b64encode((reg["username"] + ":invalid-password-123").encode()).decode()
|
||||
statuses, replicas, times = [], set(), []
|
||||
for _ in range(6):
|
||||
r = client.call(
|
||||
"POST", "/api/v1/auth/sessions", [401, 429], headers={"Authorization": "Basic " + bad}
|
||||
)
|
||||
statuses.append(r["status"])
|
||||
times.append(r["elapsed"] * 1000)
|
||||
replicas.add(r["headers"]["X-Instance-Id"])
|
||||
if r["status"] == 429:
|
||||
assert int(r["headers"]["Retry-After"]) > 0
|
||||
assert 429 in statuses and 401 in statuses and len(replicas) >= 2
|
||||
too_big = client.call(
|
||||
"POST", "/api/v1/users", 413, raw=b"x" * 66000, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
assert too_big["status"] == 413
|
||||
return {
|
||||
"statuses": statuses,
|
||||
"replicas": len(replicas),
|
||||
"latency_ms": times,
|
||||
"outsider_denied": True,
|
||||
"logout_revoked": True,
|
||||
"oversize_json_denied": True,
|
||||
}
|
||||
|
||||
|
||||
def rotate(client, temp):
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
old = temp / "old"
|
||||
shutil.copytree(LOCAL / "tls/worker-client", old)
|
||||
compose("stop", "worker")
|
||||
accepted = []
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for _ in range(5):
|
||||
att, data = client.attachment(alice, chat)
|
||||
accepted.append((att, data))
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--revoke", "worker")
|
||||
checked(sys.executable, "scripts/pki.py", "--rotate", "worker-client")
|
||||
password = LOCAL / "broker_worker_password"
|
||||
password.write_text(secrets.token_hex(24) + "\n")
|
||||
checked(sys.executable, "scripts/bootstrap.py")
|
||||
# Load the changed ACL without restarting durable Valkey or erasing its stream.
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"sh",
|
||||
"-ec",
|
||||
"""p="$(awk '/^user operator / {print substr($NF,2)}' /run/secrets/broker.acl)"
|
||||
export REDISCLI_AUTH="$p"
|
||||
valkey-cli --user operator ACL LOAD | grep -q '^OK$'
|
||||
valkey-cli --user operator CLIENT KILL USER worker >/dev/null""",
|
||||
)
|
||||
compose("restart", "broker-tls", "otel-collector-tls")
|
||||
result = isolated("old", {"old": old})
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
for attachment, original in accepted:
|
||||
item = client.wait_ready(alice, attachment)
|
||||
file = client.call("GET", f"/api/v1/attachments/{item['id']}/content", actor=alice)
|
||||
assert hashlib.sha256(file["bytes"]).digest() == hashlib.sha256(original).digest()
|
||||
return {
|
||||
"old_identity": result,
|
||||
"accepted": len(accepted),
|
||||
"ready_with_original": len(accepted),
|
||||
"elapsed_seconds": round(time.monotonic() - start, 3),
|
||||
"broker_acl_reloaded": True,
|
||||
"images_rebuilt": False,
|
||||
"new_worker": worker_probe(),
|
||||
}
|
||||
|
||||
|
||||
def check_sealed(client):
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
start = time.monotonic()
|
||||
checked(sys.executable, "scripts/bao_operator.py", "--seal")
|
||||
try:
|
||||
deadline = start + 42
|
||||
status = 200
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 503:
|
||||
break
|
||||
time.sleep(1)
|
||||
assert status == 503, "API did not stop serving after the credential grace period"
|
||||
unavailable = time.monotonic() - start
|
||||
finally:
|
||||
checked(sys.executable, "scripts/bao_operator.py")
|
||||
deadline = time.monotonic() + 22
|
||||
while time.monotonic() < deadline:
|
||||
status, _ = response("https://localhost:8443/health/ready", context(CA))
|
||||
if status == 200:
|
||||
return {
|
||||
"unready_after_seconds": round(unavailable, 3),
|
||||
"recovered_after_seconds": round(time.monotonic() - start, 3),
|
||||
}
|
||||
time.sleep(1)
|
||||
raise AssertionError("API readiness did not recover after unseal")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--inside", choices=("tls", "worker", "old"))
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--rotate", action="store_true")
|
||||
parser.add_argument("--seal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.inside:
|
||||
print(
|
||||
json.dumps(
|
||||
{"tls": audit_tls, "worker": worker_scope, "old": old_identity}[args.inside]()
|
||||
)
|
||||
)
|
||||
return
|
||||
if not args.output or args.output.exists():
|
||||
parser.error("choose a new --output")
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(CA))
|
||||
with tempfile.TemporaryDirectory(dir=LOCAL) as folder:
|
||||
tmp = Path(folder)
|
||||
tmp.chmod(0o755)
|
||||
rogue = tmp / "rogue"
|
||||
rogue.mkdir(mode=0o755)
|
||||
checked(
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"ec",
|
||||
"-pkeyopt",
|
||||
"ec_paramgen_curve:P-256",
|
||||
"-nodes",
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
"/CN=worker-client",
|
||||
"-addext",
|
||||
"extendedKeyUsage=clientAuth",
|
||||
"-keyout",
|
||||
str(rogue / "service.key"),
|
||||
"-out",
|
||||
str(rogue / "service.crt"),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
(rogue / "service.key").chmod(0o644)
|
||||
identities = {
|
||||
name: LOCAL / "tls" / name
|
||||
for name in ("lb-client", "worker-client", "probe-client", "grafana-client")
|
||||
}
|
||||
result = {
|
||||
"tls": isolated("tls", {**identities, "rogue": rogue}),
|
||||
"worker_scope": worker_probe(),
|
||||
"api": check_api(client),
|
||||
"external_hostname": verify_hostname(),
|
||||
}
|
||||
print("PASS: verified TLS and distinct service privileges", flush=True)
|
||||
if args.rotate:
|
||||
result["worker_compromise"] = rotate(client, tmp)
|
||||
print("PASS: old identity denied; all accepted uploads recovered", flush=True)
|
||||
if args.seal:
|
||||
result["sealed"] = check_sealed(client)
|
||||
print(
|
||||
"PASS: sealed OpenBao makes API unready within a bounded grace period", flush=True
|
||||
)
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = "from pathlib import Path; print(Path('/tmp/api-instance-id').read_text())"
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Use an operator connection only to seed a legacy hash and an expired test session."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def inside():
|
||||
import psycopg
|
||||
|
||||
from messenger.db_roles import admin_conninfo
|
||||
|
||||
fixture = json.load(sys.stdin)
|
||||
with psycopg.connect(admin_conninfo()) as db:
|
||||
if fixture["mode"] == "seed":
|
||||
db.execute(
|
||||
"UPDATE users SET password_hash=%s WHERE id=%s",
|
||||
(fixture["hash"], fixture["user"]),
|
||||
)
|
||||
elif fixture["mode"] == "check":
|
||||
row = db.execute(
|
||||
"SELECT password_hash FROM users WHERE id=%s", (fixture["user"],)
|
||||
).fetchone()
|
||||
assert row and row[0].startswith("$argon2id$")
|
||||
assert row[0].split("$")[3] == "m=19456,t=2,p=1"
|
||||
db.execute(
|
||||
"UPDATE sessions SET expires_at=created_at+interval '1 millisecond' "
|
||||
"WHERE token_hash=%s",
|
||||
(bytes.fromhex(fixture["digest"]),),
|
||||
)
|
||||
else:
|
||||
raise AssertionError("Unknown fixture mode")
|
||||
|
||||
|
||||
def db_fixture(**data):
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"-v",
|
||||
f"{Path(__file__).resolve()}:/test/verify_sessions.py:ro",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"migrate",
|
||||
"/test/verify_sessions.py",
|
||||
"--inside",
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
input=json.dumps(data),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if "--inside" in sys.argv:
|
||||
inside()
|
||||
return
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(ROOT / "secrets/tls/trust/ca.crt"))
|
||||
actor = client.user("legacy")
|
||||
reg = actor["registration"]
|
||||
legacy = PasswordHasher(memory_cost=65536, time_cost=3, parallelism=4).hash(reg["password"])
|
||||
db_fixture(mode="seed", user=actor["id"], hash=legacy)
|
||||
auth = base64.b64encode((reg["username"] + ":" + reg["password"]).encode()).decode()
|
||||
session = client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
schema="Session",
|
||||
headers={"Authorization": "Basic " + auth},
|
||||
)["json"]
|
||||
actor["token"] = session["token"]
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
db_fixture(
|
||||
mode="check", user=actor["id"], digest=hashlib.sha256(actor["token"].encode()).hexdigest()
|
||||
)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
print("PASS: successful login rehashed old Argon2id parameters; expired session rejected.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify the real Grafana data paths over authenticated HTTPS."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CA = ROOT / "secrets/tls/trust/ca.crt"
|
||||
|
||||
|
||||
def grafana(uid, path, **params):
|
||||
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
||||
basic = base64.b64encode(("admin:" + password).encode()).decode()
|
||||
url = f"https://localhost:3007/api/datasources/proxy/uid/{uid}{path}"
|
||||
if params:
|
||||
url += "?" + urlencode(params)
|
||||
request = Request(url, headers={"Authorization": "Basic " + basic})
|
||||
with urlopen(request, context=ssl.create_default_context(cafile=CA), timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def until(fn):
|
||||
end = time.monotonic() + 35
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
value = fn()
|
||||
except HTTPError as exc:
|
||||
if exc.code not in (404, 502, 503):
|
||||
raise
|
||||
else:
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("Timed out waiting for Grafana datasource results")
|
||||
|
||||
|
||||
def logs(service, field, value):
|
||||
# The query is restricted to a generated ID, which is validated by the probe.
|
||||
return grafana(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
query=f'{{service_name="messenger-{service}"}} | {field}="{value}"',
|
||||
start=str(time.time_ns() - 120_000_000_000),
|
||||
limit=10,
|
||||
)["data"]["result"]
|
||||
|
||||
|
||||
def main():
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
client = Client("https://localhost:8443", str(CA))
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
marker = "probe_" + uuid.uuid4().hex
|
||||
client.message(actor, chat, marker)
|
||||
upload = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename={marker}.png",
|
||||
202,
|
||||
actor=actor,
|
||||
headers={"Content-Type": "image/png"},
|
||||
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
schema="Attachment",
|
||||
)
|
||||
client.wait_ready(actor, upload["json"])
|
||||
request_id = str(uuid.UUID(upload["headers"]["X-Request-Id"]))
|
||||
api_logs = until(lambda: logs("api", "request_id", request_id))
|
||||
stream = api_logs[0]
|
||||
value = stream["values"][0]
|
||||
trace_id = (value[2] if len(value) > 2 else stream["stream"])["trace_id"]
|
||||
assert len(trace_id) == 32 and all(ch in "0123456789abcdef" for ch in trace_id)
|
||||
worker_logs = until(lambda: logs("worker", "trace_id", trace_id))
|
||||
trace = until(lambda: grafana("tempo", "/api/traces/" + trace_id))
|
||||
collected = json.dumps([api_logs, worker_logs, trace])
|
||||
assert "jobs.consume" in collected and "s3.put" in collected and "db.transaction" in collected
|
||||
for forbidden in (marker, actor["registration"]["password"], actor["token"]):
|
||||
assert forbidden not in collected
|
||||
until(
|
||||
lambda: grafana("prometheus", "/api/v1/query", query="sum(messenger_http_requests_total)")[
|
||||
"data"
|
||||
]["result"]
|
||||
)
|
||||
print("PASS: three Grafana datasources over mTLS; correlated payloads are clean.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+16
-16
@@ -1,19 +1,19 @@
|
||||
# Модель угроз — заполнить
|
||||
# Модель угроз ЛР7
|
||||
|
||||
## Активы и акторы
|
||||
Активы: пароли и сессии, сообщения и изображения, задания, ключи сервисов, CA и телеметрия. Границы: клиент ↔ LB, LB ↔ API, приложение ↔ хранилища и брокер, приложение ↔ OpenBao, эксплуатационный контур ↔ оператор. Оператор и хост входят в доверенную основу. Атакующий может быть клиентом вне чата, процессом в сети Compose, обладателем сертификата другой роли или обладателем прав worker.
|
||||
|
||||
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
|
||||
| Угроза | Контроль | Воспроизведение |
|
||||
| --- | --- | --- |
|
||||
| Перехват логина или токена | TLS с CA и SAN на ingress; hostname проверяется | `make test-security` |
|
||||
| Пользователь подделывает X-User-Id и читает чужое вложение | Bearer, проверка членства, приватный bucket | `make test`, `scripts/security_probe.py` |
|
||||
| Прямой запрос к API в обход LB | API требует client cert и точный URI SAN балансировщика | mTLS негативные пробы API |
|
||||
| Доверенный сертификат другой роли используется для ingest или broker | Allowlist ролей на прокси, CRL, Valkey ACL, закрытый loopback hop | mTLS негативные пробы и TCP-проверка внутренних портов |
|
||||
| Перебор паролей нагружает KDF | Argon2id, общий бюджет логина в PostgreSQL, 429/Retry-After, два KDF слота на реплику | один логин через обе реплики, измерение и rehash в `verify_sessions.py` |
|
||||
| Украденная сессия остаётся после logout/expiry | SHA-256 случайного токена в PostgreSQL, отзыв и абсолютный TTL | публичный smoke и fixture истёкшей сессии |
|
||||
| Скомпрометированный worker получает админские ключи | Раздельные scoped DB/S3/Valkey/OpenBao identities, клиентский ключ только для worker | worker probe проверяет allow/deny на каждом контуре |
|
||||
| Старый ключ используют после отзыва | CRL + reload серверов, отключение старого cert-auth leaf, замена Valkey секрета и завершение старых соединений | `security_probe.py --rotate`, повторное TLS-подключение старым leaf |
|
||||
| OpenBao sealed или пропала связь | Ограниченный grace, готовность API и пауза потребления после expiry | `security_probe.py --seal` |
|
||||
| Изображение истощает память или время | Лимиты размеров и пикселей, child process, cgroup, предельное время, lease и fence | smoke и унаследованный pipeline из ЛР5/ЛР6 |
|
||||
| Секрет попадает в телеметрию или артефакт | Allowlist полей, игнорирование файлов секретов, закрытый экспорт, проверки содержимого | трасса и лог синтетических запросов, аудит staged diff |
|
||||
|
||||
## Границы доверия
|
||||
|
||||
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
|
||||
|
||||
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO | TODO |
|
||||
|
||||
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
|
||||
|
||||
## Lifecycle и инциденты
|
||||
|
||||
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
|
||||
Доступ worker к законным объектам и задачам сохраняется в пределах его роли. Если из памяти отдельно извлечены пароль PostgreSQL или ключ S3, отзыв сертификата не отзывает эти статические значения; оператор меняет их отдельно. Модель не покрывает компрометацию хоста, CA, root token, потерю диска и кражу уже расшифрованных данных. Ограничение на фиксированное минутное окно демонстрирует общий бюджет на стенде; оно допускает пограничный всплеск и не является распределённой системой защиты от DDoS.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""One-process implementation of the lab 1 messenger contract."""
|
||||
@@ -0,0 +1,40 @@
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
|
||||
import uvicorn
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.tls import IngressProtocol
|
||||
|
||||
|
||||
def main():
|
||||
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
|
||||
logging.basicConfig(level=level, format="%(message)s")
|
||||
config = uvicorn.Config(
|
||||
"messenger.app:create_app",
|
||||
factory=True,
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
http=IngressProtocol,
|
||||
access_log=False,
|
||||
proxy_headers=False,
|
||||
log_level=level.lower(),
|
||||
timeout_keep_alive=2,
|
||||
timeout_graceful_shutdown=5,
|
||||
ssl_keyfile="/run/tls/server/service.key",
|
||||
ssl_certfile="/run/tls/server/service.crt",
|
||||
ssl_ca_certs="/run/tls/trust/ca.crt",
|
||||
ssl_cert_reqs=ssl.CERT_REQUIRED,
|
||||
)
|
||||
with telemetry.active("api"):
|
||||
config.load()
|
||||
config.ssl.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
config.ssl.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF
|
||||
config.ssl.load_verify_locations(cafile="/run/tls/trust/revoked.crl")
|
||||
uvicorn.Server(config).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,390 @@
|
||||
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Annotated
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import INVALID_SPAN, SpanKind
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.attachments import Attachments
|
||||
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User, invalid, not_found
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.schemas import (
|
||||
AttachmentView,
|
||||
ChatPage,
|
||||
ChatView,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
MessagePage,
|
||||
MessageView,
|
||||
SessionView,
|
||||
UserView,
|
||||
canonical_uuid,
|
||||
)
|
||||
from messenger.secrets_provider import ServiceLease
|
||||
from messenger.service import Messenger
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
if scope.get("path") in {"/health/live", "/health/ready"}:
|
||||
await self.request(scope, receive, send, INVALID_SPAN)
|
||||
return
|
||||
with telemetry.span("http.request", context=Context(), kind=SpanKind.SERVER) as span:
|
||||
await self.request(scope, receive, send, span)
|
||||
|
||||
async def request(self, scope, receive, send, span):
|
||||
request_id = str(uuid4())
|
||||
scope.setdefault("state", {})["request_id"] = request_id
|
||||
start, status, response_started = perf_counter(), 500, False
|
||||
|
||||
async def with_id(message: Message):
|
||||
nonlocal status, response_started
|
||||
if message["type"] == "http.response.start":
|
||||
response_started = True
|
||||
status = message["status"]
|
||||
message["headers"] = [
|
||||
pair
|
||||
for pair in message.get("headers", [])
|
||||
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
|
||||
] + [
|
||||
(b"x-request-id", request_id.encode()),
|
||||
(b"x-instance-id", self.instance_id.encode()),
|
||||
]
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
app = scope.get("app")
|
||||
lease = getattr(app.state, "service_lease", None) if app is not None else None
|
||||
if lease is not None and not lease.valid() and scope["path"] != "/health/live":
|
||||
await error_response(
|
||||
Request(scope), 503, "unavailable", "Доступ сервиса временно недоступен"
|
||||
)(scope, receive, with_id)
|
||||
return
|
||||
if scope["method"] in {"POST", "PUT", "PATCH"} and not scope["path"].endswith(
|
||||
"/attachments"
|
||||
):
|
||||
body = bytearray()
|
||||
more = True
|
||||
while more:
|
||||
item = await receive()
|
||||
if item["type"] == "http.disconnect":
|
||||
return
|
||||
body.extend(item.get("body", b""))
|
||||
if len(body) > 65536:
|
||||
await error_response(
|
||||
Request(scope), 413, "payload_too_large", "Слишком большой запрос"
|
||||
)(scope, receive, with_id)
|
||||
return
|
||||
more = item.get("more_body", False)
|
||||
original_receive = receive
|
||||
first = True
|
||||
|
||||
async def limited_receive():
|
||||
nonlocal first
|
||||
if first:
|
||||
first = False
|
||||
return {"type": "http.request", "body": bytes(body), "more_body": False}
|
||||
return await original_receive()
|
||||
|
||||
receive = limited_receive
|
||||
await self.app(scope, receive, with_id)
|
||||
except Exception as exc:
|
||||
# Do not let Uvicorn print exception values that could contain user data.
|
||||
if response_started:
|
||||
raise
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
response = error_response(
|
||||
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
await response(scope, receive, with_id)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
if route not in {"/health/live", "/health/ready"}:
|
||||
telemetry.http(
|
||||
span, scope["method"], route, status, perf_counter() - start, request_id
|
||||
)
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request, status: int, code: str, message: str, headers: dict | None = None
|
||||
) -> JSONResponse:
|
||||
identity = getattr(request.state, "request_id", str(uuid4()))
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": identity}},
|
||||
headers={"X-Request-Id": identity, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
settings = settings or Settings.from_env()
|
||||
repository = PostgresRepository(
|
||||
settings.database_url,
|
||||
settings.pool_size,
|
||||
settings.idempotency_ttl,
|
||||
settings.read_autocommit,
|
||||
)
|
||||
cache = SessionCache(settings.cache_url)
|
||||
store = ObjectStore(settings)
|
||||
authentication = AuthService(repository, cache, settings.session_ttl)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
repository.open()
|
||||
if os.getenv("BAO_ROLE") == "api":
|
||||
Path("/tmp/api-instance-id").write_text(app.state.instance_id)
|
||||
app.state.ready = True
|
||||
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
|
||||
try:
|
||||
with ServiceLease(settings) as lease:
|
||||
app.state.service_lease = lease
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
repository.close()
|
||||
cache.close()
|
||||
store.close()
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
|
||||
app.state.messenger = Messenger(repository)
|
||||
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
|
||||
app.state.repository = repository
|
||||
app.state.authentication = authentication
|
||||
app.state.cache = cache
|
||||
app.state.ready = False
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, exc: DomainError):
|
||||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException):
|
||||
code = "not_found" if exc.status_code == 404 else "invalid_request"
|
||||
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
|
||||
def service(request: Request) -> Messenger:
|
||||
return request.app.state.messenger
|
||||
|
||||
def token_digest(request: Request) -> bytes:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized()
|
||||
return bearer_digest(request.headers.get("authorization"))
|
||||
|
||||
def identity(
|
||||
digest: Annotated[bytes, Depends(token_digest)],
|
||||
) -> Identity:
|
||||
return authentication.identify(digest)
|
||||
|
||||
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
|
||||
return current.user
|
||||
|
||||
async def credentials(request: Request) -> tuple[str, str]:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized("Basic")
|
||||
result = basic_credentials(request.headers.get("authorization"))
|
||||
if await request.body():
|
||||
raise invalid("Вход не принимает тело запроса")
|
||||
return result
|
||||
|
||||
def resource_id(chat_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(chat_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID чата") from None
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
for key in ("limit", "cursor"):
|
||||
if len(request.query_params.getlist(key)) > 1:
|
||||
raise invalid("Параметр пагинации указан несколько раз")
|
||||
raw = request.query_params.get("limit", "50")
|
||||
try:
|
||||
if not re.fullmatch(r"[0-9]+", raw):
|
||||
raise ValueError
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid("limit должен быть целым от 1 до 100") from None
|
||||
return limit, request.query_params.get("cursor")
|
||||
|
||||
def json_input(request: Request):
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
|
||||
|
||||
def message_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
|
||||
raise invalid("Требуется корректный Idempotency-Key")
|
||||
return values[0]
|
||||
|
||||
CurrentUser = Annotated[User, Depends(actor)]
|
||||
Service = Annotated[Messenger, Depends(service)]
|
||||
ChatId = Annotated[str, Depends(resource_id)]
|
||||
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.ready:
|
||||
raise DomainError(503, "unavailable", "Приложение не готово")
|
||||
repository.ready()
|
||||
store.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=UserView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def register(body: CreateUser):
|
||||
return authentication.register(body.username, body.display_name, body.password)
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
|
||||
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
|
||||
return authentication.login(*value)
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(current: Annotated[Identity, Depends(identity)]):
|
||||
authentication.logout(current)
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=UserView)
|
||||
def me(current: CurrentUser):
|
||||
return current
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=ChatView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
|
||||
return messenger.create_chat(current, body.title, body.member_ids)
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def chats(current: CurrentUser, messenger: Service, page: Pagination):
|
||||
return messenger.chats(current, *page)
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
|
||||
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
|
||||
return messenger.chat(current, chat)
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=MessageView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def send_message(
|
||||
body: CreateMessage,
|
||||
current: CurrentUser,
|
||||
chat: ChatId,
|
||||
messenger: Service,
|
||||
key: Annotated[str, Depends(message_key)],
|
||||
):
|
||||
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
|
||||
return messenger.messages(current, chat, *page)
|
||||
|
||||
def attachment_id(attachment_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(attachment_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID вложения") from None
|
||||
|
||||
AttachmentId = Annotated[str, Depends(attachment_id)]
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentView)
|
||||
async def upload(request: Request, current: CurrentUser, chat: ChatId):
|
||||
return await app.state.attachments.upload(request, current, chat)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
|
||||
def attachment(current: CurrentUser, id: AttachmentId):
|
||||
return app.state.attachments.metadata(current, id)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def content(request: Request, current: CurrentUser, id: AttachmentId):
|
||||
row = app.state.attachments.record(current, id)
|
||||
if row["storage_state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
values = request.query_params.getlist("variant")
|
||||
if (
|
||||
len(values) > 1
|
||||
or values
|
||||
and values[0] not in ("original", "thumbnail", "crop", "optimized")
|
||||
):
|
||||
raise invalid("Некорректный вариант вложения")
|
||||
if values and values[0] != "original":
|
||||
row = next((v for v in row["variants"] if v["name"] == values[0]), None)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
row = {**row, "filename": values[0]}
|
||||
body = store.download(row)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := body.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Bounded upload: verified staging and transactional job intent precede HTTP 202."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
from anyio import to_thread
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from messenger.domain import DomainError, invalid, new_id, not_found
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import attachment_value
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not value.strip()
|
||||
or len(value) > 255
|
||||
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
|
||||
):
|
||||
raise invalid("Некорректное имя файла")
|
||||
return value
|
||||
|
||||
|
||||
def image_type(file, maximum):
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > maximum:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError
|
||||
media = "image/png" if image.format == "PNG" else "image/jpeg"
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load() # Reject truncated pixel streams, not just an intact header.
|
||||
file.seek(0)
|
||||
return media
|
||||
except Image.DecompressionBombError, Image.DecompressionBombWarning:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
|
||||
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, repository, messenger, store, settings):
|
||||
self.repository, self.messenger, self.store = repository, messenger, store
|
||||
self.settings = settings
|
||||
self.slots = asyncio.Semaphore(settings.upload_slots)
|
||||
|
||||
async def upload(self, request, actor, chat_id):
|
||||
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
|
||||
values = request.query_params.getlist("filename")
|
||||
if len(values) != 1:
|
||||
raise invalid("Требуется одно имя файла")
|
||||
name = filename(values[0])
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media not in ("image/png", "image/jpeg"):
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal():
|
||||
raise invalid("Некорректный Content-Length")
|
||||
if int(length) > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
try:
|
||||
await asyncio.wait_for(self.slots.acquire(), timeout=5)
|
||||
except TimeoutError:
|
||||
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
digest.update(chunk)
|
||||
await to_thread.run_sync(file.write, chunk)
|
||||
except TimeoutError:
|
||||
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
|
||||
except ClientDisconnect:
|
||||
raise invalid("Загрузка прервана") from None
|
||||
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
|
||||
if actual != media:
|
||||
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
|
||||
id = new_id()
|
||||
key = "staging/" + id
|
||||
await to_thread.run_sync(Pipeline(self.repository, self.settings).reserve, id, key)
|
||||
await to_thread.run_sync(
|
||||
self.store.put, key, file, size, actual, digest.hexdigest()
|
||||
)
|
||||
# A failure before this COMMIT leaves only an unaccepted staging orphan.
|
||||
# The same transaction creates attachment, job and outbox intent.
|
||||
row = await to_thread.run_sync(
|
||||
Pipeline(self.repository, self.settings).accept,
|
||||
id,
|
||||
chat_id,
|
||||
actor.id,
|
||||
name,
|
||||
actual,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
key,
|
||||
)
|
||||
return attachment_value(row)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def record(self, actor, id):
|
||||
row = self.repository.attachment(id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
self.messenger.chat(actor, str(row["chat_id"]))
|
||||
return row
|
||||
|
||||
def metadata(self, actor, id):
|
||||
return attachment_value(self.record(actor, id))
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User
|
||||
from messenger.repository import PostgresRepository
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> DomainError:
|
||||
challenge = scheme + ' realm="messenger"'
|
||||
if scheme == "Basic":
|
||||
challenge += ', charset="UTF-8"'
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется действительная идентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def basic_credentials(header: str | None) -> tuple[str, str]:
|
||||
try:
|
||||
if header is None or len(header) > 2048:
|
||||
raise ValueError
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise ValueError
|
||||
if not 12 <= len(password) <= 128 or not password.strip():
|
||||
raise ValueError
|
||||
return username, password
|
||||
except ValueError, UnicodeError, binascii.Error:
|
||||
raise unauthorized("Basic") from None
|
||||
|
||||
|
||||
def bearer_digest(header: str | None) -> bytes:
|
||||
if header:
|
||||
parts = header.split(" ")
|
||||
if len(parts) == 2 and parts[0].lower() == "bearer":
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
|
||||
return hashlib.sha256(parts[1].encode()).digest()
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Identity:
|
||||
user: User
|
||||
token_hash: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
|
||||
self.repository, self.cache, self.ttl = repository, cache, ttl
|
||||
memory = int(os.getenv("ARGON2_MEMORY_KIB", "19456"))
|
||||
iterations = int(os.getenv("ARGON2_ITERATIONS", "2"))
|
||||
parallelism = int(os.getenv("ARGON2_PARALLELISM", "1"))
|
||||
if not (19456 <= memory <= 65536 and 2 <= iterations <= 5 and 1 <= parallelism <= 4):
|
||||
raise ValueError("Argon2 budget outside the supported range")
|
||||
self.hasher = PasswordHasher(
|
||||
memory_cost=memory, time_cost=iterations, parallelism=parallelism
|
||||
)
|
||||
self.kdf_slots = BoundedSemaphore(2)
|
||||
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
@contextmanager
|
||||
def kdf(self):
|
||||
if not self.kdf_slots.acquire(timeout=0.5):
|
||||
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.kdf_slots.release()
|
||||
|
||||
def register(self, username: str, display_name: str, password: str) -> User:
|
||||
with self.kdf():
|
||||
encoded = self.hasher.hash(password)
|
||||
return self.repository.add_user(username, display_name, encoded)
|
||||
|
||||
def login(self, username: str, password: str) -> Session:
|
||||
wait = self.repository.login_retry_after(hashlib.sha256(username.encode()).hexdigest())
|
||||
if wait:
|
||||
raise DomainError(
|
||||
429, "rate_limited", "Слишком много попыток входа", {"Retry-After": str(wait)}
|
||||
)
|
||||
credentials = self.repository.credentials(username)
|
||||
encoded = credentials[1] if credentials else self.dummy_hash
|
||||
try:
|
||||
with self.kdf():
|
||||
self.hasher.verify(encoded, password)
|
||||
if credentials and self.hasher.check_needs_rehash(encoded):
|
||||
updated = self.hasher.hash(password)
|
||||
else:
|
||||
updated = None
|
||||
except VerificationError, InvalidHashError:
|
||||
raise unauthorized("Basic") from None
|
||||
if credentials is None:
|
||||
raise unauthorized("Basic")
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).digest()
|
||||
user = credentials[0]
|
||||
if updated:
|
||||
self.repository.update_password(user.id, encoded, updated)
|
||||
expires = self.repository.create_session(digest, user.id, self.ttl)
|
||||
return Session(token, "Bearer", expires, user)
|
||||
|
||||
def identify(self, digest: bytes) -> Identity:
|
||||
session = self.repository.active_session(digest)
|
||||
if session is None:
|
||||
raise unauthorized()
|
||||
user_id = str(session["user_id"])
|
||||
user = self.cache.get(digest, user_id)
|
||||
if user is None:
|
||||
user = self.repository.get_user(user_id)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
|
||||
self.cache.put(digest, user, ttl)
|
||||
return Identity(user, digest)
|
||||
|
||||
def logout(self, identity: Identity):
|
||||
self.repository.revoke_session(identity.token_hash)
|
||||
self.cache.delete(identity.token_hash)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
STREAM = "attachment-jobs"
|
||||
GROUP = "image-workers-v1"
|
||||
DLQ = "invalid-jobs"
|
||||
|
||||
|
||||
class Job(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
schema_version: Literal[1]
|
||||
job_id: str
|
||||
type: Literal["attachment.process.v1"]
|
||||
attachment_id: str
|
||||
pipeline_version: int = Field(ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: str
|
||||
traceparent: str | None = Field(
|
||||
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
||||
)
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def strict_carrier(cls, value):
|
||||
if isinstance(value, dict):
|
||||
if type(value.get("schema_version")) is not int:
|
||||
raise ValueError("Integer schema version required")
|
||||
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
|
||||
raise ValueError("Optional carrier fields must be strings when provided")
|
||||
return value
|
||||
|
||||
@field_validator("job_id", "attachment_id")
|
||||
@classmethod
|
||||
def uuid(cls, value):
|
||||
if str(UUID(value)) != value:
|
||||
raise ValueError("Canonical UUID required")
|
||||
return value
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def timestamp(cls, value):
|
||||
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
|
||||
raise ValueError("RFC3339 required")
|
||||
datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return value
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, settings):
|
||||
self.client = redis.Redis.from_url(
|
||||
settings.broker_url,
|
||||
decode_responses=True,
|
||||
socket_timeout=2,
|
||||
socket_connect_timeout=1,
|
||||
retry_on_timeout=False,
|
||||
)
|
||||
|
||||
def group(self):
|
||||
try:
|
||||
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if "BUSYGROUP" not in str(exc):
|
||||
raise
|
||||
|
||||
def publish(self, job):
|
||||
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
|
||||
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
|
||||
|
||||
def ack(self, id):
|
||||
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
|
||||
self.client.eval(
|
||||
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
|
||||
1,
|
||||
STREAM,
|
||||
GROUP,
|
||||
id,
|
||||
)
|
||||
|
||||
def quarantine(self, id, reason):
|
||||
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
|
||||
with self.client.pipeline(transaction=True) as tx:
|
||||
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
|
||||
tx.xack(STREAM, GROUP, id)
|
||||
tx.xdel(STREAM, id)
|
||||
tx.execute()
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Disposable session-user cache; PostgreSQL always decides token validity."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from messenger.domain import User
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str):
|
||||
self.client = Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
max_connections=8,
|
||||
decode_responses=True,
|
||||
retry_on_timeout=False,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session-user:" + digest.hex()
|
||||
|
||||
def get(self, digest: bytes, user_id: str) -> User | None:
|
||||
try:
|
||||
raw = self.client.get(self.key(digest))
|
||||
if raw:
|
||||
value = User(**json.loads(raw))
|
||||
if value.id == user_id:
|
||||
return value
|
||||
except RedisError, ValueError, TypeError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
|
||||
return None
|
||||
|
||||
def put(self, digest: bytes, user: User, ttl: int):
|
||||
try:
|
||||
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
|
||||
|
||||
def delete(self, digest: bytes):
|
||||
try:
|
||||
self.client.delete(self.key(digest))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Dry-run by default. Retention GC protects all active jobs and every published object."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.pipeline import event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def collect(db, store, retention, apply=False):
|
||||
if retention < 300:
|
||||
raise ValueError("Retention must be at least 300 seconds")
|
||||
counts = {"scanned": 0, "eligible": 0, "deleted": 0, "protected": 0}
|
||||
# Serialize collectors; API handoff is bounded < retention and all owned keys have UUID prefix.
|
||||
with db.connection() as lock:
|
||||
if not lock.execute("SELECT pg_try_advisory_lock(2301005) AS ok").fetchone()["ok"]:
|
||||
return {**counts, "busy": True}
|
||||
try:
|
||||
for page in store.client.get_paginator("list_objects_v2").paginate(Bucket=store.bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
counts["scanned"] += 1
|
||||
key = obj["Key"]
|
||||
parts = key.split("/")
|
||||
try:
|
||||
if (
|
||||
parts[0] not in ("staging", "objects")
|
||||
or str(UUID(parts[1])) != parts[1]
|
||||
):
|
||||
continue
|
||||
except ValueError, IndexError:
|
||||
continue
|
||||
if (datetime.now(UTC) - obj["LastModified"]).total_seconds() < retention:
|
||||
continue
|
||||
with db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
if (
|
||||
reservation
|
||||
and reservation["state"] == "uploading"
|
||||
and reservation["expires_at"] > datetime.now(UTC)
|
||||
):
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
# This row lock serializes replay/new-version with deletion eligibility.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (parts[1],)
|
||||
).fetchone()
|
||||
active = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state IN ('queued','processing') LIMIT 1",
|
||||
(parts[1],),
|
||||
).fetchone()
|
||||
published = (
|
||||
a
|
||||
and a["storage_state"] == "ready"
|
||||
and (
|
||||
key == a["object_key"]
|
||||
or any(v["object_key"] == key for v in a["variants"])
|
||||
)
|
||||
)
|
||||
recent_failed = conn.execute(
|
||||
"SELECT 1 FROM attachment_jobs WHERE attachment_id=%s AND "
|
||||
"state='failed' "
|
||||
"AND updated_at>clock_timestamp()-(%s*interval '1 second') LIMIT 1",
|
||||
(parts[1], retention),
|
||||
).fetchone()
|
||||
if active or published or recent_failed:
|
||||
counts["protected"] += 1
|
||||
continue
|
||||
head = store.head(key)
|
||||
if (
|
||||
not head
|
||||
or head["ETag"] != obj["ETag"]
|
||||
or head["ContentLength"] != obj["Size"]
|
||||
or (datetime.now(UTC) - head["LastModified"]).total_seconds()
|
||||
< retention
|
||||
):
|
||||
continue
|
||||
counts["eligible"] += 1
|
||||
if apply:
|
||||
if reservation and reservation["state"] == "uploading":
|
||||
conn.execute(
|
||||
"UPDATE upload_reservations SET state='abandoned' WHERE id=%s",
|
||||
(parts[1],),
|
||||
)
|
||||
store.client.delete_object(Bucket=store.bucket, Key=key)
|
||||
counts["deleted"] += 1
|
||||
rows = conn.execute(
|
||||
"UPDATE attachment_jobs SET source_expired=true WHERE "
|
||||
"source_key=%s "
|
||||
"AND state='failed' RETURNING job_id",
|
||||
(key,),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
event(conn, row["job_id"], "source_expired")
|
||||
finally:
|
||||
lock.execute("SELECT pg_advisory_unlock(2301005)")
|
||||
return counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
print(json.dumps(collect(db, store, settings.staging_retention, args.apply)))
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Versioned keyset position, bound to a list and its resource, never an offset."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.domain import HistoryKey, invalid
|
||||
|
||||
|
||||
def encode_cursor(scope: str, position: HistoryKey) -> str:
|
||||
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
|
||||
return (
|
||||
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not value or len(value) > 1024:
|
||||
raise ValueError
|
||||
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
|
||||
data = json.loads(raw)
|
||||
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
|
||||
raise ValueError
|
||||
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
|
||||
raise ValueError
|
||||
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
|
||||
raise ValueError
|
||||
stamp = datetime.fromisoformat(data["created_at"])
|
||||
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
|
||||
raise ValueError
|
||||
identity = str(UUID(data["id"]))
|
||||
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
|
||||
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
|
||||
raise invalid("Некорректный cursor или другая область пагинации") from None
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Operator-only migrations and PostgreSQL least-privilege grants."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.migrate import migrate
|
||||
|
||||
SECRET_DIR = Path("/run/secrets")
|
||||
ROLES = ("api", "worker", "relay", "reconciler")
|
||||
|
||||
|
||||
def admin_conninfo():
|
||||
return make_conninfo(
|
||||
host="postgres",
|
||||
port=5432,
|
||||
dbname="messenger",
|
||||
user="messenger",
|
||||
password=(SECRET_DIR / "postgres_password").read_text().strip(),
|
||||
sslmode="verify-full",
|
||||
sslrootcert="/run/tls/trust/ca.crt",
|
||||
)
|
||||
|
||||
|
||||
def configure(conn):
|
||||
conn.execute("REVOKE ALL ON DATABASE messenger FROM PUBLIC")
|
||||
conn.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC")
|
||||
for role in ROLES:
|
||||
identifier = sql.Identifier(role)
|
||||
if not conn.execute("SELECT 1 FROM pg_roles WHERE rolname=%s", (role,)).fetchone():
|
||||
conn.execute(sql.SQL("CREATE ROLE {} LOGIN").format(identifier))
|
||||
password = sql.Literal((SECRET_DIR / ("db_" + role + "_password")).read_text().strip())
|
||||
conn.execute(sql.SQL("ALTER ROLE {} PASSWORD {}").format(identifier, password))
|
||||
conn.execute(sql.SQL("GRANT CONNECT ON DATABASE messenger TO {}").format(identifier))
|
||||
conn.execute(sql.SQL("GRANT USAGE ON SCHEMA public TO {}").format(identifier))
|
||||
conn.execute("""
|
||||
GRANT SELECT ON schema_migrations TO api;
|
||||
GRANT SELECT, INSERT ON users, chats, chat_members, messages, message_attachments TO api;
|
||||
GRANT UPDATE(password_hash) ON users TO api;
|
||||
GRANT SELECT, INSERT, DELETE ON sessions, message_requests TO api;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON login_budget TO api;
|
||||
GRANT SELECT, INSERT ON attachments, attachment_jobs TO api;
|
||||
GRANT UPDATE(updated_at) ON attachments TO api;
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE ON job_outbox, upload_reservations TO api;
|
||||
|
||||
GRANT SELECT ON attachments, attachment_jobs TO worker, relay, reconciler;
|
||||
GRANT SELECT, INSERT, UPDATE ON job_outbox TO worker, relay;
|
||||
GRANT UPDATE(storage_state, object_key, variants, error_code, updated_at)
|
||||
ON attachments TO worker;
|
||||
GRANT UPDATE(storage_state, error_code, updated_at) ON attachments TO relay;
|
||||
GRANT UPDATE(state, attempt, fence, lease_until, error_code, next_attempt_at, updated_at)
|
||||
ON attachment_jobs TO worker;
|
||||
GRANT UPDATE(state, lease_until, error_code, next_attempt_at, updated_at)
|
||||
ON attachment_jobs TO relay;
|
||||
GRANT UPDATE(source_expired) ON attachment_jobs TO reconciler;
|
||||
GRANT UPDATE(updated_at) ON attachments TO reconciler;
|
||||
GRANT SELECT ON upload_reservations TO reconciler;
|
||||
GRANT UPDATE(state) ON upload_reservations TO reconciler;
|
||||
|
||||
GRANT INSERT ON pipeline_audit TO api, worker, relay;
|
||||
GRANT USAGE ON SEQUENCE pipeline_audit_sequence_seq TO api, worker, relay;
|
||||
""")
|
||||
|
||||
|
||||
def main():
|
||||
conninfo = admin_conninfo()
|
||||
migrate(conninfo, Path("/app/migrations"))
|
||||
with psycopg.connect(conninfo, connect_timeout=5) as conn:
|
||||
configure(conn)
|
||||
print("Scoped PostgreSQL roles configured.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Immutable domain values; no dependency on HTTP or a storage technology."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message: str = "Некорректный запрос") -> DomainError:
|
||||
return DomainError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
status: str = "ready"
|
||||
variants: tuple = ()
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
type HistoryKey = tuple[str, str]
|
||||
|
||||
|
||||
def history_key(item: Chat | Message) -> HistoryKey:
|
||||
return item.created_at, item.id
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Operator-only, single-use fault barriers. Disabled in the normal Compose stack."""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from messenger.domain import DomainError
|
||||
from messenger.pipeline import event
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def checkpoint(db, settings, job, phase):
|
||||
if not settings.fault_injection:
|
||||
return
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE pipeline_faults SET remaining=remaining-1 WHERE job_id=%s AND phase=%s "
|
||||
"AND remaining>0 RETURNING action",
|
||||
(job, phase),
|
||||
).fetchone()
|
||||
if row:
|
||||
event(conn, job, "fault_injected", phase=phase, action=row["action"], pid=os.getpid())
|
||||
if not row:
|
||||
return
|
||||
match row["action"]:
|
||||
case "crash":
|
||||
os._exit(87)
|
||||
case "hang":
|
||||
time.sleep(settings.job_timeout * 3)
|
||||
case "transient":
|
||||
raise DomainError(503, "unavailable", "Synthetic transient failure")
|
||||
case "permanent":
|
||||
raise PermanentFailure
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Pure, versioned image transforms. No network, filenames, or EXIF in output."""
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
|
||||
def fit(image, maximum):
|
||||
scale = max(image.size)
|
||||
if scale <= maximum:
|
||||
return image.copy()
|
||||
size = tuple(max(1, n * maximum // scale) for n in image.size)
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def variants(source, version):
|
||||
if version not in (1, 2):
|
||||
raise ValueError("Unsupported pipeline version")
|
||||
with Image.open(source) as raw:
|
||||
oriented = ImageOps.exif_transpose(raw)
|
||||
rgba = oriented.convert("RGBA")
|
||||
image = Image.new("RGB", rgba.size, "white")
|
||||
image.paste(rgba, mask=rgba.getchannel("A"))
|
||||
rgba.close()
|
||||
oriented.close()
|
||||
try:
|
||||
for name, output, format, options in (
|
||||
("thumbnail", fit(image, 256), "JPEG", {"quality": 85, "subsampling": 0}),
|
||||
(
|
||||
"crop",
|
||||
ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS),
|
||||
"JPEG",
|
||||
{"quality": 85, "subsampling": 0},
|
||||
),
|
||||
(
|
||||
"optimized",
|
||||
fit(image, 1280),
|
||||
"WEBP",
|
||||
{"quality": 82 if version == 1 else 76, "method": 4},
|
||||
),
|
||||
):
|
||||
with output, io.BytesIO() as buffer:
|
||||
output.save(buffer, format=format, exif=b"", **options)
|
||||
yield (
|
||||
name,
|
||||
buffer.getvalue(),
|
||||
output.size,
|
||||
"image/webp" if format == "WEBP" else "image/jpeg",
|
||||
)
|
||||
finally:
|
||||
image.close()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Transactional, checksummed migrations, serialized across concurrent starts."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from messenger.settings import secret
|
||||
|
||||
|
||||
def migrate(conninfo: str, directory: Path):
|
||||
with psycopg.connect(conninfo, connect_timeout=5) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
|
||||
)
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
content = path.read_text()
|
||||
checksum = hashlib.sha256(content.encode()).hexdigest()
|
||||
current = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
|
||||
).fetchone()
|
||||
if current:
|
||||
if current[0] != checksum:
|
||||
raise RuntimeError("Applied migration checksum changed: " + path.stem)
|
||||
continue
|
||||
conn.execute(content)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
|
||||
(path.stem, checksum),
|
||||
)
|
||||
print("Applied migration " + path.stem)
|
||||
|
||||
|
||||
def main():
|
||||
migrate(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Explicit local operator actions, never exposed as a public HTTP endpoint."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("failed", "replay", "new-version"))
|
||||
parser.add_argument("attachment_id", nargs="?")
|
||||
args = parser.parse_args()
|
||||
settings = Settings.from_env()
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
try:
|
||||
if args.action == "failed":
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT job_id,attachment_id,pipeline_version,attempt,error_code,"
|
||||
"source_expired FROM attachment_jobs WHERE state='failed' "
|
||||
"ORDER BY updated_at LIMIT 100"
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=str))
|
||||
else:
|
||||
if not args.attachment_id:
|
||||
parser.error("attachment_id is required")
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT j.source_key,a.object_key,a.storage_state FROM attachment_jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id AND "
|
||||
"a.pipeline_version=j.pipeline_version "
|
||||
"WHERE a.id=%s",
|
||||
(args.attachment_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
parser.error("Unknown attachment")
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
key = row["object_key"] if row["storage_state"] == "ready" else row["source_key"]
|
||||
if not store.head(key):
|
||||
parser.error("Source is no longer retained")
|
||||
finally:
|
||||
store.close()
|
||||
job = Pipeline(db, settings).replay(
|
||||
args.attachment_id, new_version=args.action == "new-version"
|
||||
)
|
||||
print(json.dumps({"job_id": job, "action": args.action}))
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Job state machine. PostgreSQL fences publication; the stream is at least once."""
|
||||
|
||||
import os
|
||||
import random
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import DomainError, new_id
|
||||
|
||||
|
||||
def event(conn, job_id, name, **details):
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_audit(job_id,event,details) VALUES (%s,%s,%s)",
|
||||
(job_id, name, Jsonb(details)),
|
||||
)
|
||||
|
||||
|
||||
def intent(conn, job_id, when=None):
|
||||
conn.execute(
|
||||
"INSERT INTO job_outbox(job_id,due_at) VALUES (%s,coalesce(%s,clock_timestamp())) "
|
||||
"ON CONFLICT(job_id) DO UPDATE SET due_at=EXCLUDED.due_at,published_at=NULL,stream_id=NULL",
|
||||
(job_id, when),
|
||||
)
|
||||
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, repository, settings):
|
||||
self.db, self.settings = repository, settings
|
||||
|
||||
def reserve(self, id, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
conn.execute("INSERT INTO upload_reservations(id,object_key) VALUES (%s,%s)", (id, key))
|
||||
|
||||
def accept(self, id, chat, owner, name, media, size, digest, key):
|
||||
with self.db.connection(write=True) as conn:
|
||||
reservation = conn.execute(
|
||||
"SELECT * FROM upload_reservations WHERE id=%s FOR UPDATE", (id,)
|
||||
).fetchone()
|
||||
if (
|
||||
not reservation
|
||||
or reservation["state"] != "uploading"
|
||||
or reservation["object_key"] != key
|
||||
or reservation["expires_at"] <= datetime.now(UTC)
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Срок приёма файла истёк")
|
||||
conn.execute("UPDATE upload_reservations SET state='accepted' WHERE id=%s", (id,))
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"object_key,storage_state) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
||||
(id, chat, owner, name, media, size, digest, key),
|
||||
).fetchone()
|
||||
job = new_id()
|
||||
trace = telemetry.carrier()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key,"
|
||||
"traceparent,tracestate) VALUES (%s,%s,1,%s,%s,%s)",
|
||||
(job, id, key, trace.get("traceparent"), trace.get("tracestate")),
|
||||
)
|
||||
intent(conn, job)
|
||||
event(conn, job, "accepted")
|
||||
return row
|
||||
|
||||
def claim(self, envelope):
|
||||
with self.db.connection(write=True) as conn:
|
||||
# All writers lock attachment before job, including publication and operator replay.
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (envelope.attachment_id,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
return "unknown", None
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE",
|
||||
(envelope.job_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
not j
|
||||
or j["attachment_id"] != a["id"]
|
||||
or j["pipeline_version"] != envelope.pipeline_version
|
||||
):
|
||||
return "unknown", None
|
||||
if a["pipeline_version"] != j["pipeline_version"] or j["state"] == "superseded":
|
||||
event(conn, j["job_id"], "stale_delivery")
|
||||
return "stale", None
|
||||
if j["state"] == "ready":
|
||||
event(conn, j["job_id"], "duplicate_ready")
|
||||
return "ready", a
|
||||
if j["state"] == "failed":
|
||||
return "failed", None
|
||||
now = datetime.now(UTC)
|
||||
if j["state"] == "processing" or j["next_attempt_at"] > now:
|
||||
return "busy", None
|
||||
if envelope.attempt != j["attempt"] + 1:
|
||||
# An older envelope cannot steal a scheduled retry or an explicit replay.
|
||||
return "obsolete_attempt", None
|
||||
fence = new_id()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='processing',attempt=attempt+1,fence=%s,"
|
||||
"lease_until=clock_timestamp()+(%s*interval '1 "
|
||||
"second'),updated_at=clock_timestamp() "
|
||||
"WHERE job_id=%s",
|
||||
(fence, self.settings.lease_seconds, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='processing',"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(a["id"],),
|
||||
)
|
||||
event(
|
||||
conn, j["job_id"], "claimed", attempt=j["attempt"] + 1, fence=fence, pid=os.getpid()
|
||||
)
|
||||
return "claimed", {
|
||||
**a,
|
||||
"job_id": str(j["job_id"]),
|
||||
"source_key": j["source_key"],
|
||||
"fence": fence,
|
||||
"attempt": j["attempt"] + 1,
|
||||
"accepted_at": j["created_at"],
|
||||
}
|
||||
|
||||
def publish(self, row, original_key, variants):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if not self.owns(a, j, row):
|
||||
event(conn, row["job_id"], "publication_fenced")
|
||||
return False
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='ready',object_key=%s,variants=%s,"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(original_key, Jsonb(variants), row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='ready',lease_until=NULL,error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(row["job_id"],),
|
||||
)
|
||||
event(conn, row["job_id"], "result_committed", version=row["pipeline_version"])
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def owns(a, j, row):
|
||||
return (
|
||||
a
|
||||
and j
|
||||
and a["pipeline_version"] == row["pipeline_version"]
|
||||
and j["state"] == "processing"
|
||||
and str(j["fence"]) == row["fence"]
|
||||
and j["lease_until"] > datetime.now(UTC)
|
||||
)
|
||||
|
||||
def failure(self, row, code, permanent=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)
|
||||
).fetchone()
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id=%s FOR NO KEY UPDATE", (row["job_id"],)
|
||||
).fetchone()
|
||||
if self.owns(a, j, row):
|
||||
return self._retry(conn, j, code, permanent)
|
||||
return "fenced"
|
||||
|
||||
def _retry(self, conn, j, code, permanent=False):
|
||||
failed = permanent or j["attempt"] >= self.settings.job_max_attempts
|
||||
state = "failed" if failed else "queued"
|
||||
if failed and not permanent:
|
||||
code = "retry_exhausted"
|
||||
# Full jitter; durable absolute time survives worker/relay restarts.
|
||||
delay = random.uniform(0.25, min(30, 2 ** min(j["attempt"], 5)))
|
||||
due = datetime.now(UTC) + timedelta(seconds=delay)
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state=%s,error_code=%s,next_attempt_at=%s,lease_until=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE job_id=%s",
|
||||
(state, code, due, j["job_id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state=%s,error_code=%s,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s",
|
||||
(state, code if failed else None, j["attachment_id"]),
|
||||
)
|
||||
event(conn, j["job_id"], state, reason=code, attempt=j["attempt"], delay_seconds=delay)
|
||||
if not failed:
|
||||
intent(conn, j["job_id"], due)
|
||||
return "failed" if failed else "retry"
|
||||
|
||||
def recover(self):
|
||||
# Bounded SKIP LOCKED batch; multiple relays. No transaction held during Pillow.
|
||||
with self.db.connection(write=True) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT a.id FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id "
|
||||
"AND j.pipeline_version=a.pipeline_version WHERE j.state='processing' "
|
||||
"AND j.lease_until<clock_timestamp() ORDER BY a.id LIMIT 32 "
|
||||
"FOR UPDATE OF a SKIP LOCKED"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
j = conn.execute(
|
||||
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"AND a.pipeline_version=j.pipeline_version WHERE a.id=%s FOR NO KEY "
|
||||
"UPDATE OF j",
|
||||
(row["id"],),
|
||||
).fetchone()
|
||||
if j["state"] == "processing" and j["lease_until"] < datetime.now(UTC):
|
||||
event(conn, j["job_id"], "lease_expired")
|
||||
self._retry(conn, j, "worker_timeout")
|
||||
|
||||
def replay(self, attachment, *, new_version=False):
|
||||
with self.db.connection(write=True) as conn:
|
||||
a = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment,)
|
||||
).fetchone()
|
||||
if not a:
|
||||
raise ValueError("Unknown attachment")
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s "
|
||||
"AND pipeline_version=%s FOR UPDATE",
|
||||
(attachment, a["pipeline_version"]),
|
||||
).fetchone()
|
||||
if new_version:
|
||||
if a["pipeline_version"] != 1:
|
||||
raise ValueError("Only pipeline v1 -> v2 is implemented")
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET state='superseded' WHERE job_id=%s "
|
||||
"AND state IN ('queued','processing','failed')",
|
||||
(j["job_id"],),
|
||||
)
|
||||
key = a["object_key"] if a["storage_state"] == "ready" else j["source_key"]
|
||||
job = new_id()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(job_id,attachment_id,pipeline_version,source_key) "
|
||||
"VALUES (%s,%s,2,%s)",
|
||||
(job, attachment, key),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET "
|
||||
"pipeline_version=2,storage_state='queued',variants='[]',"
|
||||
"error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "new_version", previous_job=str(j["job_id"]))
|
||||
else:
|
||||
if j["state"] != "failed" or j["source_expired"]:
|
||||
raise ValueError("Replay requires failed job with retained source")
|
||||
job = str(j["job_id"])
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"state='queued',attempt=0,fence=NULL,error_code=NULL,"
|
||||
"next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE "
|
||||
"job_id=%s",
|
||||
(job,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET storage_state='queued',error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(attachment,),
|
||||
)
|
||||
event(conn, job, "operator_replay")
|
||||
intent(conn, job)
|
||||
return job
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
|
||||
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.broker import Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import checkpoint
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.secrets_provider import ServiceLease
|
||||
from messenger.settings import Settings
|
||||
|
||||
|
||||
def publish_one(db, broker, settings):
|
||||
with db.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at,"
|
||||
"j.traceparent,j.tracestate "
|
||||
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
|
||||
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
|
||||
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
|
||||
).fetchone()
|
||||
if not row:
|
||||
return False
|
||||
with telemetry.span("jobs.publish", context=telemetry.context(row), kind=SpanKind.PRODUCER):
|
||||
job = Job(
|
||||
schema_version=1,
|
||||
type=telemetry.JOB_TYPE,
|
||||
job_id=str(row["job_id"]),
|
||||
attachment_id=str(row["attachment_id"]),
|
||||
pipeline_version=row["pipeline_version"],
|
||||
attempt=row["attempt"] + 1,
|
||||
created_at=stamp(row["created_at"]),
|
||||
**telemetry.carrier(),
|
||||
)
|
||||
stream_id = broker.publish(job)
|
||||
telemetry.event("job_published", job_id=job.job_id, attempt=job.attempt)
|
||||
# Commit-on-return may be lost here; repeat sends the same logical job_id.
|
||||
checkpoint(db, settings, row["job_id"], "after_publish")
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
|
||||
(stream_id, row["job_id"]),
|
||||
)
|
||||
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
|
||||
return True
|
||||
|
||||
|
||||
def sample_backlog(db):
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS depth, "
|
||||
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age "
|
||||
"FROM attachment_jobs WHERE state IN ('queued','processing')"
|
||||
).fetchone()
|
||||
telemetry.gauge("messenger.queue.depth", row["depth"], queue="attachment-jobs")
|
||||
telemetry.gauge(
|
||||
"messenger.queue.oldest", max(0, float(row["age"])), unit="s", queue="attachment-jobs"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
settings = Settings.from_env()
|
||||
with telemetry.active("relay"), ServiceLease(settings) as lease:
|
||||
run(settings, lease)
|
||||
|
||||
|
||||
def run(settings, lease):
|
||||
db = PostgresRepository(settings.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker = Broker(settings)
|
||||
pipeline = Pipeline(db, settings)
|
||||
running = True
|
||||
next_sample = 0.0
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
if not lease.valid():
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
try:
|
||||
if telemetry.runtime is not None and time.monotonic() >= next_sample:
|
||||
sample_backlog(db)
|
||||
next_sample = time.monotonic() + 2
|
||||
pipeline.recover()
|
||||
sent = publish_one(db, broker, settings)
|
||||
Path("/tmp/relay-heartbeat").touch()
|
||||
if not sent:
|
||||
time.sleep(0.2)
|
||||
except redis.RedisError, DomainError:
|
||||
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,376 @@
|
||||
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import (
|
||||
Attachment,
|
||||
Chat,
|
||||
DomainError,
|
||||
HistoryKey,
|
||||
Message,
|
||||
User,
|
||||
invalid,
|
||||
new_id,
|
||||
)
|
||||
|
||||
|
||||
def stamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_value(row) -> User:
|
||||
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
|
||||
|
||||
|
||||
def chat_value(row) -> Chat:
|
||||
return Chat(
|
||||
str(row["id"]),
|
||||
row["title"],
|
||||
tuple(str(x) for x in row["member_ids"]),
|
||||
stamp(row["created_at"]),
|
||||
)
|
||||
|
||||
|
||||
def message_value(row) -> Message:
|
||||
return Message(
|
||||
str(row["id"]),
|
||||
str(row["chat_id"]),
|
||||
str(row["sender_id"]),
|
||||
row["text"],
|
||||
stamp(row["created_at"]),
|
||||
tuple(str(x) for x in row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
def attachment_value(row) -> Attachment:
|
||||
return Attachment(
|
||||
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
|
||||
row["filename"],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
row["sha256"],
|
||||
stamp(row["created_at"]),
|
||||
stamp(row["updated_at"]),
|
||||
row["storage_state"],
|
||||
tuple(
|
||||
{k: v for k, v in item.items() if k != "object_key"} for item in row.get("variants", [])
|
||||
),
|
||||
row.get("error_code"),
|
||||
)
|
||||
|
||||
|
||||
class Repository(Protocol):
|
||||
def get_user(self, user_id: str) -> User | None: ...
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
|
||||
def get_chat(self, chat_id: str) -> Chat | None: ...
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message: ...
|
||||
def page_messages(
|
||||
self, chat_id: str, after: HistoryKey | None, limit: int
|
||||
) -> list[Message]: ...
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
|
||||
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(
|
||||
self,
|
||||
conninfo: str,
|
||||
pool_size: int = 8,
|
||||
idempotency_ttl: int = 86400,
|
||||
read_autocommit: bool = True,
|
||||
):
|
||||
self.read_autocommit = read_autocommit
|
||||
self.idempotency_ttl = idempotency_ttl
|
||||
self.pool = ConnectionPool(
|
||||
conninfo,
|
||||
min_size=1,
|
||||
max_size=pool_size,
|
||||
timeout=1,
|
||||
max_waiting=32,
|
||||
open=False,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": True,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
|
||||
},
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=15)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=5)
|
||||
|
||||
@contextmanager
|
||||
def connection(self, *, write: bool = False):
|
||||
with telemetry.span(
|
||||
"db.transaction" if write else "db.read", **{"db.system.name": "postgresql"}
|
||||
):
|
||||
with self._connection(write=write) as conn:
|
||||
yield conn
|
||||
|
||||
@contextmanager
|
||||
def _connection(self, *, write):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
transaction = (
|
||||
conn.transaction() if write or not self.read_autocommit else nullcontext()
|
||||
)
|
||||
with transaction:
|
||||
telemetry.pool(self.pool)
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
errors.QueryCanceled,
|
||||
errors.LockNotAvailable,
|
||||
):
|
||||
telemetry.pool(self.pool)
|
||||
telemetry.record("messenger.db.errors", 1)
|
||||
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS n FROM schema_migrations "
|
||||
"WHERE version IN "
|
||||
"('001_initial','002_idempotency','003_attachments',"
|
||||
"'004_pipeline','005_upload_handoff','006_trace_context','007_login_budget')"
|
||||
).fetchone()
|
||||
if row["n"] != 7:
|
||||
raise DomainError(503, "unavailable", "Схема ещё не готова")
|
||||
|
||||
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (id,username,display_name,password_hash) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return user_value(row)
|
||||
except errors.UniqueViolation:
|
||||
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
|
||||
|
||||
def credentials(self, username: str):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
return (user_value(row), row["password_hash"]) if row else None
|
||||
|
||||
def update_password(self, user_id, previous, encoded):
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"UPDATE users SET password_hash=%s WHERE id=%s AND password_hash=%s",
|
||||
(encoded, user_id, previous),
|
||||
)
|
||||
|
||||
def login_retry_after(self, username_hash):
|
||||
with self.connection(write=True) as conn:
|
||||
epoch = conn.execute(
|
||||
"SELECT floor(extract(epoch FROM clock_timestamp()))::bigint AS seconds"
|
||||
).fetchone()["seconds"]
|
||||
window = epoch // 60 * 60
|
||||
conn.execute("DELETE FROM login_budget WHERE window_start < %s", (window - 60,))
|
||||
for scope, limit in (("global", 90), (username_hash, 5)):
|
||||
row = conn.execute(
|
||||
"INSERT INTO login_budget(scope,window_start,attempts) VALUES (%s,%s,1) "
|
||||
"ON CONFLICT(scope,window_start) DO UPDATE "
|
||||
"SET attempts=least(login_budget.attempts+1,%s) RETURNING attempts",
|
||||
(scope, window, limit + 1),
|
||||
).fetchone()
|
||||
if row["attempts"] > limit:
|
||||
return 60 - (epoch - window)
|
||||
return 0
|
||||
|
||||
def get_user(self, user_id: str) -> User | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
return user_value(row) if row else None
|
||||
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
[(row["id"], member, i) for i, member in enumerate(member_ids)],
|
||||
)
|
||||
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
|
||||
except errors.ForeignKeyViolation:
|
||||
raise invalid("Указан неизвестный участник") from None
|
||||
|
||||
def get_chat(self, chat_id: str) -> Chat | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
|
||||
).fetchone()
|
||||
return chat_value(row) if row else None
|
||||
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
|
||||
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (user_id, *after, limit) if after else (user_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
|
||||
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [chat_value(row) for row in rows]
|
||||
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
return message_value(row)
|
||||
|
||||
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
|
||||
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (chat_id, *after, limit) if after else (chat_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s"
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [message_value(row) for row in rows]
|
||||
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message:
|
||||
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
payload = json.dumps(
|
||||
{"text": text, "attachment_ids": list(attachment_ids)},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
digest = hashlib.sha256(payload.encode()).digest()
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
previous = conn.execute(
|
||||
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
|
||||
"JOIN messages m ON m.id=r.response_message_id "
|
||||
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
|
||||
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
|
||||
(sender_id, chat_id, key),
|
||||
).fetchone()
|
||||
if previous:
|
||||
if previous["payload_hash"] != digest:
|
||||
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
|
||||
return message_value(previous)
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
|
||||
"AND chat_id=%s AND idempotency_key=%s",
|
||||
(sender_id, chat_id, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
|
||||
(chat_id, list(attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise DomainError(404, "not_found", "Объект не найден")
|
||||
if any(row["storage_state"] == "failed" for row in rows):
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
message = message_value(
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
)
|
||||
for position, attachment in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
|
||||
(message.id, chat_id, attachment, position),
|
||||
)
|
||||
message = Message(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
attachment_ids,
|
||||
)
|
||||
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
|
||||
return message
|
||||
|
||||
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests "
|
||||
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
|
||||
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
|
||||
)
|
||||
|
||||
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO sessions (token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
|
||||
(token_hash, user_id, ttl),
|
||||
).fetchone()
|
||||
return stamp(row["expires_at"])
|
||||
|
||||
def active_session(self, token_hash: bytes):
|
||||
with self.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
|
||||
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
|
||||
def revoke_session(self, token_hash: bytes):
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
|
||||
|
||||
def attachment(self, id):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Strict lab 5 wire types; password never belongs to a response User."""
|
||||
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def canonical_uuid(value: str) -> str:
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
|
||||
raise ValueError("Ожидается UUID")
|
||||
return str(UUID(value))
|
||||
|
||||
|
||||
def nonblank(value: str) -> str:
|
||||
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
|
||||
raise ValueError("Ожидается непустая строка Unicode")
|
||||
return value
|
||||
|
||||
|
||||
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
|
||||
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(strict=True, extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
display_name: ShortText
|
||||
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def valid_username(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Недопустимое имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: ShortText
|
||||
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_content(self):
|
||||
if (
|
||||
not self.text.strip()
|
||||
and not self.attachment_ids
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
|
||||
or len(set(self.attachment_ids)) != len(self.attachment_ids)
|
||||
):
|
||||
raise ValueError("Некорректное содержимое сообщения")
|
||||
return self
|
||||
|
||||
|
||||
class View(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
||||
|
||||
|
||||
class UserView(View):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionView(View):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserView
|
||||
|
||||
|
||||
class ChatView(View):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessageView(View):
|
||||
attachment_ids: list[str]
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(View):
|
||||
items: list[ChatView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class MessagePage(View):
|
||||
items: list[MessageView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class AttachmentView(View):
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Certificate login with short-lived, path-scoped OpenBao tokens."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
from threading import Event, Thread
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
def load_service_secrets(role):
|
||||
address = os.getenv("BAO_ADDR", "https://openbao:8200").rstrip("/")
|
||||
if not address.startswith("https://"):
|
||||
raise ValueError("OpenBao requires HTTPS")
|
||||
context = ssl.create_default_context(cafile="/run/tls/trust/ca.crt")
|
||||
context.load_cert_chain("/run/tls/client/service.crt", "/run/tls/client/service.key")
|
||||
|
||||
def call(path, *, data=None, token=None):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
request = Request(
|
||||
f"{address}/v1/{path}",
|
||||
data=json.dumps(data).encode() if data is not None else None,
|
||||
headers=headers,
|
||||
)
|
||||
with urlopen(request, context=context, timeout=3) as response:
|
||||
body = response.read(32768)
|
||||
return json.loads(body) if body else {}
|
||||
|
||||
token = None
|
||||
try:
|
||||
token = call("auth/cert/login", data={"name": role})["auth"]["client_token"]
|
||||
return call(f"services/data/{role}", token=token)["data"]["data"]
|
||||
except OSError, ValueError, KeyError:
|
||||
raise RuntimeError("Scoped service credentials are unavailable") from None
|
||||
finally:
|
||||
if token:
|
||||
try:
|
||||
call("auth/token/revoke-self", data={}, token=token)
|
||||
except OSError:
|
||||
pass # Its finite TTL still applies if the operator sealed the service.
|
||||
|
||||
|
||||
class ServiceLease:
|
||||
"""Reauthenticate periodically. Stop work after a bounded outage or a secret change."""
|
||||
|
||||
def __init__(self, settings):
|
||||
self.role = os.getenv("BAO_ROLE")
|
||||
self.settings = settings
|
||||
self.grace = int(os.getenv("BAO_CREDENTIAL_GRACE", "30"))
|
||||
self.interval = int(os.getenv("BAO_REFRESH_INTERVAL", "5"))
|
||||
if not 2 <= self.interval < self.grace <= 120:
|
||||
raise ValueError("Invalid OpenBao refresh or grace interval")
|
||||
self.deadline = time.monotonic() + self.grace
|
||||
self.stop = Event()
|
||||
self.thread = Thread(target=self.refresh, name="openbao-refresh", daemon=True)
|
||||
|
||||
def __enter__(self):
|
||||
if self.role:
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
if self.role:
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=4)
|
||||
|
||||
def valid(self):
|
||||
return not self.role or time.monotonic() < self.deadline
|
||||
|
||||
def refresh(self):
|
||||
while not self.stop.wait(self.interval):
|
||||
try:
|
||||
values = load_service_secrets(self.role)
|
||||
except RuntimeError:
|
||||
continue
|
||||
if any(getattr(self.settings, key) != value for key, value in values.items()):
|
||||
# This process still holds the old pools. The operator recreates it after rotation.
|
||||
self.deadline = 0
|
||||
continue
|
||||
self.deadline = time.monotonic() + self.grace
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from messenger.cursor import decode_cursor, encode_cursor
|
||||
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
|
||||
from messenger.repository import Repository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page[T]:
|
||||
items: list[T]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class Messenger:
|
||||
def __init__(self, repository: Repository):
|
||||
self.repository = repository
|
||||
|
||||
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
|
||||
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
|
||||
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
|
||||
return self.repository.add_chat(title, tuple(member_ids))
|
||||
|
||||
def chat(self, actor: User, chat_id: str) -> Chat:
|
||||
chat = self.repository.get_chat(chat_id)
|
||||
if chat is None or actor.id not in chat.member_ids:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
|
||||
scope = "chats:" + actor.id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
|
||||
|
||||
def send(
|
||||
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
|
||||
) -> Message:
|
||||
self.chat(actor, chat_id)
|
||||
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
|
||||
|
||||
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
|
||||
self.chat(actor, chat_id)
|
||||
scope = "messages:" + chat_id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
|
||||
|
||||
@staticmethod
|
||||
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
|
||||
visible = items[:limit]
|
||||
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
|
||||
return Page(visible, cursor)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Explicit configuration; credentials are read from untracked secret files."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from messenger.secrets_provider import load_service_secrets
|
||||
|
||||
|
||||
def secret(name: str, default: str, optional=False) -> str:
|
||||
path = Path(os.getenv(name, default))
|
||||
return path.read_text().strip() if path.exists() or not optional else ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 8
|
||||
idempotency_ttl: int = 86400
|
||||
read_autocommit: bool = True
|
||||
|
||||
s3_endpoint: str = "http://127.0.0.1:59006"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key: str = field(default="", repr=False)
|
||||
s3_secret_key: str = field(default="", repr=False)
|
||||
max_upload_bytes: int = 10485760
|
||||
max_image_pixels: int = 40000000
|
||||
upload_slots: int = 1
|
||||
|
||||
broker_url: str = field(default="redis://broker:6379/0", repr=False)
|
||||
worker_concurrency: int = 2
|
||||
job_max_attempts: int = 5
|
||||
job_timeout: int = 20
|
||||
staging_retention: int = 86400
|
||||
fault_injection: bool = False
|
||||
|
||||
@property
|
||||
def lease_seconds(self):
|
||||
return self.job_timeout + 3
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
|
||||
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if idempotency_ttl < 86400:
|
||||
raise ValueError("Idempotency retention must be at least 24 hours")
|
||||
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
|
||||
raise ValueError("Invalid session lifetime or connection pool size")
|
||||
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
|
||||
raise ValueError("Invalid upload limits")
|
||||
concurrency = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
||||
attempts = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
timeout = int(os.getenv("JOB_TIMEOUT_SECONDS", "20"))
|
||||
retention = int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))
|
||||
if not 1 <= concurrency <= 4 or not 1 <= attempts <= 10 or not 2 <= timeout <= 120:
|
||||
raise ValueError("Invalid worker limits")
|
||||
if retention < 300:
|
||||
raise ValueError("Staging retention must be at least 300 seconds")
|
||||
role = os.getenv("BAO_ROLE")
|
||||
scoped = load_service_secrets(role) if role else {}
|
||||
|
||||
def credential(key, name, path, *, optional=False):
|
||||
if role:
|
||||
return scoped[key]
|
||||
return secret(name, path, optional)
|
||||
|
||||
return cls(
|
||||
credential("database_url", "DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
credential("cache_url", "CACHE_URL_FILE", "/run/secrets/cache_url"),
|
||||
ttl,
|
||||
size,
|
||||
idempotency_ttl,
|
||||
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
|
||||
os.getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
os.getenv("S3_REGION", "us-east-1"),
|
||||
os.getenv("S3_BUCKET", "messenger-attachments"),
|
||||
credential(
|
||||
"s3_access_key",
|
||||
"S3_ACCESS_KEY_ID_FILE",
|
||||
"/run/secrets/s3_access_key_id",
|
||||
optional=True,
|
||||
),
|
||||
credential(
|
||||
"s3_secret_key",
|
||||
"S3_SECRET_ACCESS_KEY_FILE",
|
||||
"/run/secrets/s3_secret_access_key",
|
||||
optional=True,
|
||||
),
|
||||
maximum,
|
||||
pixels,
|
||||
broker_url=scoped["broker_url"] if role else "rediss://broker:6380/0",
|
||||
worker_concurrency=concurrency,
|
||||
job_max_attempts=attempts,
|
||||
job_timeout=timeout,
|
||||
staging_retention=retention,
|
||||
fault_injection=os.getenv("ENABLE_FAULT_INJECTION", "0") == "1",
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
|
||||
|
||||
import os
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.domain import DomainError
|
||||
|
||||
|
||||
def unavailable():
|
||||
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
verify=os.getenv("TLS_CA_FILE") or True,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
config=Config(
|
||||
connect_timeout=0.5,
|
||||
read_timeout=2,
|
||||
max_pool_connections=8,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def put(self, key, file, size, media, digest):
|
||||
with telemetry.span("s3.put"):
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=file,
|
||||
ContentLength=size,
|
||||
ContentType=media,
|
||||
Metadata={"sha256": digest},
|
||||
)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def head(self, key):
|
||||
with telemetry.span("s3.head"):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if str(exc.response.get("Error", {}).get("Code")) in (
|
||||
"404",
|
||||
"NoSuchKey",
|
||||
"NotFound",
|
||||
):
|
||||
return None
|
||||
raise unavailable() from None
|
||||
except BotoCoreError:
|
||||
raise unavailable() from None
|
||||
|
||||
def download(self, row):
|
||||
with telemetry.span("s3.download"):
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
|
||||
if response["ContentLength"] != row["size_bytes"]:
|
||||
response["Body"].close()
|
||||
raise unavailable()
|
||||
return response["Body"]
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Application telemetry with bounded export and an explicit attribute allowlist."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from importlib.metadata import version
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry._logs import LogRecord, SeverityNumber
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
|
||||
HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"))
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
runtime = None
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, role, instance, metrics):
|
||||
resource_attrs = Resource(
|
||||
{
|
||||
"service.name": f"messenger-{role}",
|
||||
"service.version": version("mtusi-messenger"),
|
||||
"service.instance.id": instance,
|
||||
"deployment.environment.name": "lab",
|
||||
}
|
||||
)
|
||||
ratio = float(os.getenv("OTEL_SAMPLE_RATIO", "1"))
|
||||
self.traces = TracerProvider(
|
||||
resource=resource_attrs, sampler=ParentBased(TraceIdRatioBased(ratio))
|
||||
)
|
||||
self.traces.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(timeout=1),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
)
|
||||
)
|
||||
self.tracer = self.traces.get_tracer("messenger")
|
||||
self.logs = LoggerProvider(resource=resource_attrs)
|
||||
self.logs.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(timeout=1),
|
||||
max_queue_size=512,
|
||||
max_export_batch_size=64,
|
||||
schedule_delay_millis=500,
|
||||
)
|
||||
)
|
||||
self.log = self.logs.get_logger("messenger")
|
||||
self.meters = None
|
||||
self.instruments = {}
|
||||
if metrics:
|
||||
self.meters = MeterProvider(
|
||||
resource=resource_attrs,
|
||||
metric_readers=[
|
||||
PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(timeout=1),
|
||||
export_interval_millis=2000,
|
||||
export_timeout_millis=1000,
|
||||
)
|
||||
],
|
||||
views=[
|
||||
View(
|
||||
instrument_name="messenger.*.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(BUCKETS),
|
||||
)
|
||||
],
|
||||
)
|
||||
self.meter = self.meters.get_meter("messenger")
|
||||
self.meter.create_observable_counter(
|
||||
"messenger.process.cpu", unit="s", callbacks=[self.cpu_seconds]
|
||||
)
|
||||
self.meter.create_observable_gauge(
|
||||
"messenger.process.rss", unit="By", callbacks=[self.rss_bytes]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def cpu_seconds(_):
|
||||
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
yield Observation(time.process_time() + usage.ru_utime + usage.ru_stime)
|
||||
|
||||
@staticmethod
|
||||
def rss_bytes(_):
|
||||
try:
|
||||
with open("/proc/self/statm", encoding="ascii") as stats:
|
||||
yield Observation(int(stats.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def shutdown(self):
|
||||
if self.meters:
|
||||
self.meters.shutdown(timeout_millis=1500)
|
||||
self.logs.shutdown()
|
||||
self.traces.shutdown()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def active(role, instance=None, *, metrics=True):
|
||||
global runtime
|
||||
if os.getenv("TELEMETRY_ENABLED", "0") != "1":
|
||||
yield
|
||||
return
|
||||
runtime = Telemetry(role, instance or uuid4().hex, metrics)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
state, runtime = runtime, None
|
||||
state.shutdown()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name, *, context=None, kind=SpanKind.INTERNAL, **attrs):
|
||||
if runtime is None:
|
||||
yield trace.INVALID_SPAN
|
||||
return
|
||||
with runtime.tracer.start_as_current_span(
|
||||
name,
|
||||
context=context,
|
||||
kind=kind,
|
||||
attributes=attrs,
|
||||
record_exception=False,
|
||||
set_status_on_exception=False,
|
||||
) as current:
|
||||
try:
|
||||
yield current
|
||||
except Exception as exc:
|
||||
current.set_attribute("error.type", type(exc).__name__)
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
raise
|
||||
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
if runtime is not None:
|
||||
PROPAGATOR.inject(result)
|
||||
return result
|
||||
|
||||
|
||||
def context(headers):
|
||||
return PROPAGATOR.extract(
|
||||
{k: headers[k] for k in ("traceparent", "tracestate") if headers.get(k)}
|
||||
)
|
||||
|
||||
|
||||
def record(name, value, *, kind="counter", unit="1", **attrs):
|
||||
if runtime is None or runtime.meters is None:
|
||||
return
|
||||
instrument = runtime.instruments.get(name)
|
||||
if instrument is None:
|
||||
create = {
|
||||
"counter": runtime.meter.create_counter,
|
||||
"histogram": runtime.meter.create_histogram,
|
||||
"gauge": runtime.meter.create_gauge,
|
||||
}[kind]
|
||||
instrument = runtime.instruments.setdefault(name, create(name, unit=unit))
|
||||
if kind == "counter":
|
||||
instrument.add(value, attrs)
|
||||
elif kind == "histogram":
|
||||
instrument.record(value, attrs)
|
||||
else:
|
||||
instrument.set(value, attrs)
|
||||
|
||||
|
||||
def duration(name, seconds, **attrs):
|
||||
record(name, seconds, kind="histogram", unit="s", **attrs)
|
||||
|
||||
|
||||
def gauge(name, value, *, unit="", **attrs):
|
||||
record(name, value, kind="gauge", unit=unit, **attrs)
|
||||
|
||||
|
||||
def pool(pool):
|
||||
if runtime is not None and runtime.meters is not None:
|
||||
stats = pool.get_stats()
|
||||
gauge(
|
||||
"messenger.db.pool.used",
|
||||
max(0, stats["pool_size"] - stats["pool_available"]),
|
||||
)
|
||||
gauge("messenger.db.pool.waiting", stats["requests_waiting"])
|
||||
|
||||
|
||||
def event(event_name, *, error=False, **attrs):
|
||||
fields = {"event": event_name, **attrs}
|
||||
current = trace.get_current_span().get_span_context()
|
||||
if current.is_valid:
|
||||
fields["trace_id"] = format(current.trace_id, "032x")
|
||||
fields["span_id"] = format(current.span_id, "016x")
|
||||
logger.log(logging.ERROR if error else logging.INFO, json.dumps(fields))
|
||||
if runtime is not None:
|
||||
runtime.log.emit(
|
||||
LogRecord(
|
||||
timestamp=time.time_ns(),
|
||||
observed_timestamp=time.time_ns(),
|
||||
severity_text="ERROR" if error else "INFO",
|
||||
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
|
||||
body=event_name,
|
||||
attributes=fields,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def http(current, method, route, status, elapsed, request_id):
|
||||
method = method if method in HTTP_METHODS else "_OTHER"
|
||||
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
|
||||
current.update_name(f"{method} {route}")
|
||||
current.set_attributes(
|
||||
{"http.request.method": method, "http.route": route, "http.response.status_code": status}
|
||||
)
|
||||
if status >= 500:
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
record("messenger.http.requests", 1, **attrs)
|
||||
duration("messenger.http.duration", elapsed, **attrs)
|
||||
event(
|
||||
"http_request",
|
||||
error=status >= 500,
|
||||
request_id=request_id,
|
||||
status=status,
|
||||
duration_ms=round(elapsed * 1000, 3),
|
||||
**attrs,
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Accept HTTP only after the TLS client proves the load balancer identity."""
|
||||
|
||||
from uvicorn.protocols.http.h11_impl import H11Protocol
|
||||
|
||||
LB_IDENTITY = ("URI", "urn:mtusi:lab7:lb-client")
|
||||
|
||||
|
||||
class IngressProtocol(H11Protocol):
|
||||
def connection_made(self, transport):
|
||||
tls = transport.get_extra_info("ssl_object")
|
||||
cert = tls.getpeercert() if tls else None
|
||||
if not cert or LB_IDENTITY not in cert.get("subjectAltName", ()):
|
||||
transport.abort()
|
||||
return
|
||||
super().connection_made(transport)
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Bounded supervisor: one disposable OS process per job, hard wall-clock and RAM limits."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import multiprocessing
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import redis
|
||||
from opentelemetry.trace import SpanKind
|
||||
from pydantic import ValidationError
|
||||
|
||||
from messenger import telemetry
|
||||
from messenger.attachments import image_type
|
||||
from messenger.broker import GROUP, STREAM, Broker, Job
|
||||
from messenger.domain import DomainError
|
||||
from messenger.faults import PermanentFailure, checkpoint
|
||||
from messenger.images import variants
|
||||
from messenger.pipeline import Pipeline, event
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.secrets_provider import ServiceLease
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
|
||||
def verify_effect(store, row):
|
||||
records = [row, *row["variants"]]
|
||||
if len(records) != 4 or {v["name"] for v in row["variants"]} != {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}:
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
for item in records:
|
||||
head = store.head(item["object_key"])
|
||||
if (
|
||||
not head
|
||||
or head["ContentLength"] != item["size_bytes"]
|
||||
or head["Metadata"].get("sha256") != item["sha256"]
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Incomplete result")
|
||||
|
||||
|
||||
def handle(db, broker, store, settings, id, payload):
|
||||
try:
|
||||
job = Job.model_validate_json(payload)
|
||||
except ValidationError:
|
||||
broker.quarantine(id, "invalid_envelope")
|
||||
return "invalid", None
|
||||
context = telemetry.context(job.model_dump(include={"traceparent", "tracestate"}))
|
||||
with telemetry.span(
|
||||
"jobs.consume",
|
||||
context=context,
|
||||
kind=SpanKind.CONSUMER,
|
||||
**{"messaging.destination.name": STREAM},
|
||||
) as span:
|
||||
outcome, age = process_job(db, broker, store, settings, id, job)
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
telemetry.event(
|
||||
"job_processed",
|
||||
job_id=job.job_id,
|
||||
attempt=job.attempt,
|
||||
outcome=outcome,
|
||||
error=outcome in {"retry", "failed"},
|
||||
)
|
||||
return outcome, age
|
||||
|
||||
|
||||
def process_job(db, broker, store, settings, id, job):
|
||||
pipeline = Pipeline(db, settings)
|
||||
outcome, row = pipeline.claim(job)
|
||||
if outcome == "busy":
|
||||
return "busy", None # Watchdog recovers the lease.
|
||||
if outcome == "unknown":
|
||||
broker.quarantine(id, "unknown_job")
|
||||
return "unknown", None
|
||||
if outcome == "ready":
|
||||
verify_effect(store, row)
|
||||
broker.ack(id)
|
||||
return "duplicate", None
|
||||
if outcome != "claimed":
|
||||
broker.ack(id) # Failed registry / obsolete version / durable retry owns this work.
|
||||
return outcome, None
|
||||
try:
|
||||
checkpoint(db, settings, job.job_id, "before_effect")
|
||||
with tempfile.TemporaryFile() as original:
|
||||
body = store.download({**row, "object_key": row["source_key"]})
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
while chunk := body.read(65536):
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise PermanentFailure
|
||||
digest.update(chunk)
|
||||
original.write(chunk)
|
||||
finally:
|
||||
body.close()
|
||||
if size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
raise PermanentFailure
|
||||
image_type(original, settings.max_image_pixels)
|
||||
prefix = f"objects/{row['id']}/v{row['pipeline_version']}/"
|
||||
original_key = prefix + "original"
|
||||
store.put(original_key, original, size, row["content_type"], digest.hexdigest())
|
||||
checkpoint(db, settings, job.job_id, "after_effect")
|
||||
records = []
|
||||
for name, data, (width, height), media in variants(original, row["pipeline_version"]):
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
key = prefix + name
|
||||
store.put(key, io.BytesIO(data), len(data), media, checksum)
|
||||
records.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": media,
|
||||
"size_bytes": len(data),
|
||||
"sha256": checksum,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": key,
|
||||
}
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
event(conn, job.job_id, "objects_written", version=row["pipeline_version"])
|
||||
committed = pipeline.publish(row, original_key, records)
|
||||
age = max(0.0, (datetime.now(UTC) - row["accepted_at"]).total_seconds())
|
||||
if committed:
|
||||
checkpoint(db, settings, job.job_id, "after_commit")
|
||||
broker.ack(id)
|
||||
return ("ready", age) if committed else ("fenced", None)
|
||||
except PermanentFailure:
|
||||
outcome = pipeline.failure(row, "invalid_image", permanent=True)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except MemoryError:
|
||||
outcome = pipeline.failure(row, "resource_limit", permanent=True)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except DomainError as exc:
|
||||
outcome = pipeline.failure(
|
||||
row,
|
||||
"invalid_image" if exc.status in (413, 415) else "storage_unavailable",
|
||||
permanent=exc.status in (413, 415),
|
||||
)
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
except redis.RedisError:
|
||||
# Result may already be committed; retain the pending entry for a verified no-op.
|
||||
raise
|
||||
except Exception:
|
||||
outcome = pipeline.failure(row, "processing_failed")
|
||||
broker.ack(id)
|
||||
return outcome, None
|
||||
|
||||
|
||||
def child(settings, id, payload, result, instance):
|
||||
start = time.monotonic()
|
||||
try:
|
||||
with telemetry.active("worker", instance, metrics=False):
|
||||
outcome, age = run_child(settings, id, payload)
|
||||
result.send((outcome, age, time.monotonic() - start))
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
|
||||
def run_child(settings, id, payload):
|
||||
# Linux hard limit includes Pillow buffers and native allocations. Supervisor kills hangs.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
db = PostgresRepository(settings.database_url, pool_size=2)
|
||||
db.open()
|
||||
broker, store = Broker(settings), ObjectStore(settings)
|
||||
try:
|
||||
return handle(db, broker, store, settings, id, payload)
|
||||
except redis.RedisError, DomainError:
|
||||
return "dependency_error", None # Leave pending; omit driver details.
|
||||
finally:
|
||||
store.close()
|
||||
broker.close()
|
||||
db.close()
|
||||
|
||||
|
||||
def observe(result, elapsed, timed_out):
|
||||
try:
|
||||
outcome, age, duration = result.recv() if result.poll() else ("crashed", None, elapsed)
|
||||
except EOFError:
|
||||
outcome, age, duration = "crashed", None, elapsed
|
||||
finally:
|
||||
result.close()
|
||||
if timed_out:
|
||||
outcome = "timeout"
|
||||
attrs = {"job_type": telemetry.JOB_TYPE, "outcome": outcome}
|
||||
telemetry.record("messenger.jobs.processed", 1, **attrs)
|
||||
telemetry.duration("messenger.jobs.duration", duration, **attrs)
|
||||
if age is not None:
|
||||
telemetry.duration("messenger.ready.duration", age, job_type=telemetry.JOB_TYPE)
|
||||
|
||||
|
||||
def main():
|
||||
instance = uuid4().hex
|
||||
settings = Settings.from_env()
|
||||
with telemetry.active("worker", instance), ServiceLease(settings) as lease:
|
||||
run(instance, settings, lease)
|
||||
|
||||
|
||||
def run(instance, settings, lease):
|
||||
broker = Broker(settings)
|
||||
consumer, cursor = uuid4().hex, "0-0"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
active, running = [], True
|
||||
|
||||
def stop(*_):
|
||||
nonlocal running
|
||||
running = False
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
while running:
|
||||
for task in active[:]:
|
||||
process, result, started = task
|
||||
elapsed = time.monotonic() - started
|
||||
alive = process.is_alive()
|
||||
if not alive or elapsed > settings.job_timeout:
|
||||
if alive:
|
||||
process.kill()
|
||||
process.join()
|
||||
observe(result, elapsed, alive)
|
||||
active.remove(task)
|
||||
process.close()
|
||||
telemetry.gauge("messenger.worker.busy", len(active))
|
||||
telemetry.gauge("messenger.worker.capacity", settings.worker_concurrency)
|
||||
Path("/tmp/worker-heartbeat").touch()
|
||||
if not lease.valid() or len(active) >= settings.worker_concurrency:
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
try:
|
||||
broker.group()
|
||||
reclaimed = broker.client.xautoclaim(
|
||||
STREAM, GROUP, consumer, settings.lease_seconds * 1000, cursor, count=1
|
||||
)
|
||||
cursor, entries = reclaimed[0], reclaimed[1]
|
||||
if not entries:
|
||||
result = broker.client.xreadgroup(
|
||||
GROUP, consumer, {STREAM: ">"}, count=1, block=200
|
||||
)
|
||||
entries = result[0][1] if result else []
|
||||
if entries:
|
||||
id, fields = entries[0]
|
||||
result, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(
|
||||
target=child, args=(settings, id, fields.get("job", ""), sender, instance)
|
||||
)
|
||||
process.start()
|
||||
sender.close()
|
||||
active.append((process, result, time.monotonic()))
|
||||
except redis.RedisError:
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
for process, result, _ in active:
|
||||
result.close()
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
broker.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user