ЛР 7 — Группа БВТ2304 — Аккуратов — на 5 #1
+14
-37
@@ -1,41 +1,18 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Local Compose settings only. Credentials and private keys live under ignored secrets/.
|
||||
APP_PORT=8443
|
||||
API_REPLICAS=2
|
||||
API_CPU_LIMIT=1.0
|
||||
API_MEMORY_LIMIT=512M
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
MAX_CONCURRENT_UPLOADS=10
|
||||
WORKER_CONCURRENCY=1
|
||||
WORKER_REPLICAS=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
RECONCILE_INTERVAL_SECONDS=900
|
||||
RECONCILE_GRACE_SECONDS=3600
|
||||
GRAFANA_PORT=3000
|
||||
OTEL_SDK_DISABLED=false
|
||||
OTEL_TRACES_SAMPLER=parentbased_traceidratio
|
||||
OTEL_TRACES_SAMPLER_ARG=1.0
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY . /app
|
||||
RUN chown -R app:app /app
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["python3", "src/server.py"]
|
||||
+3
-3
@@ -31,8 +31,8 @@
|
||||
|
||||
Исходные README/контракт/публичные tests сохраняются, реализация переносится студентом между лабами через собственные исходники и миграции. Исправление общей спецификации публикуйте отдельным изменением, объясняя влияние на все затронутые лабораторные.
|
||||
|
||||
## Проверка этой версии skeleton
|
||||
## Проверка текущей реализации
|
||||
|
||||
На 2026-09-10 выполнены структурные проверки и полная валидация всех семи OpenAPI/JSON Schema, проверка локальных Markdown-ссылок/YAML и декодирование PNG fixture. HTTP-клиенты проверены на временном изолированном тестовом стенде: позитивный smoke 1–7, отрицательные варианты неправильного sender, pagination, ACL, logout, bytes и пустого API; также пути persistence/failover/load и внешнего TLS.
|
||||
На 2026-11-04 выполнены `make check`, полная OpenAPI-валидация, тесты парольных примитивов, проверка Python 3.9 syntax grammar, `docker compose config --quiet` и сборка API image. Реальный `MutualTLSServer` проверен с локальными сертификатами: запрос без client certificate, с foreign CA и с неразрешённым URI SAN отвергнуты; разрешённая LB identity принята.
|
||||
|
||||
Временный стенд не включён в студенческие каталоги и не является эталонным решением. Реальное приложение, Compose, PostgreSQL/S3/broker и mTLS/OpenBao в skeleton отсутствуют: их работоспособность не заявляется. Docker CLI в среде подготовки не был доступен. Это проверка материалов и тестовых драйверов, не приёмка будущих решений студентов.
|
||||
Полный Compose smoke, OpenBao runtime allow/deny, persistence/failover и нагрузочный прогон следует выполнять на изолированном стенде после operator init/unseal и создания локальных credentials.
|
||||
|
||||
@@ -4,7 +4,10 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-security
|
||||
.PHONY: help check validate test test-primitives test-internal-mtls test-login-limit test-openbao-acl test-worker-isolation test-worker-rotation test-leaf-rotation test-telemetry-redaction test-argon-budget load benchmark-observability \
|
||||
test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults \
|
||||
test-pipeline-replay test-outbox-recovery test-worker-recovery test-reconcile-integration \
|
||||
test-upload-concurrency test-persistence test-failover test-security
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -18,9 +21,70 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-primitives:
|
||||
$(PYTHON) -m unittest discover -s tests -p 'test_*.py'
|
||||
|
||||
test-internal-mtls:
|
||||
docker compose exec -T api python3 /app/tests/internal_mtls.py
|
||||
|
||||
test-login-limit:
|
||||
$(PYTHON) tests/login_rate_limit.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-openbao-acl:
|
||||
$(PYTHON) tests/openbao_acl.py --ca-file "$(CURDIR)/secrets/pki/ca.crt"
|
||||
|
||||
test-worker-isolation:
|
||||
docker compose exec -T worker python3 /app/tests/worker_isolation.py
|
||||
|
||||
test-worker-rotation:
|
||||
@test -n "$(ROOT_TOKEN_FILE)" || (echo "Set ROOT_TOKEN_FILE to an operator-only root-token file"; exit 2)
|
||||
$(PYTHON) tests/worker_identity_rotation.py --ca-file "$(CURDIR)/secrets/pki/ca.crt" --root-token-file "$(ROOT_TOKEN_FILE)"
|
||||
|
||||
test-leaf-rotation:
|
||||
$(PYTHON) tests/leaf_certificate_rotation.py
|
||||
|
||||
test-telemetry-redaction:
|
||||
$(PYTHON) tests/telemetry_redaction.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-argon-budget:
|
||||
docker compose exec -T api python3 /app/tests/argon2_budget.py
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
benchmark-observability:
|
||||
$(PYTHON) student-tests/track1_benchmark.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency-race:
|
||||
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile:
|
||||
$(PYTHON) student-tests/test_reconcile.py
|
||||
|
||||
test-worker:
|
||||
$(PYTHON) student-tests/test_worker.py
|
||||
|
||||
test-pipeline-faults:
|
||||
$(PYTHON) student-tests/test_pipeline_faults.py
|
||||
|
||||
test-pipeline-replay:
|
||||
$(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-outbox-recovery:
|
||||
$(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-worker-recovery:
|
||||
$(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile-integration:
|
||||
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-upload-concurrency:
|
||||
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -2,15 +2,13 @@
|
||||
|
||||
Защитить работающий мессенджер на границах доверия и показать защиту отрицательными проверками: чужой доступ, неверный сертификат, недостаточные права и отзыв credentials.
|
||||
|
||||
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
|
||||
Основа приложения и проверок перенесена из lab6. В этой работе сохранены REST API, PostgreSQL, S3-вложения, RabbitMQ worker и OpenTelemetry-стенд; внешний ingress работает по HTTPS, связь LB→API — по mTLS с проверкой URI SAN, пароли хешируются Argon2id, а рабочие secrets выдаются сервисам через OpenBao AppRole.
|
||||
|
||||
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
|
||||
|
||||
## Откуда начинаем
|
||||
## Состав стенда
|
||||
|
||||
Перенесите минимум lab6. Бизнес-API сохраняется, внешний endpoint теперь HTTPS. Новые секреты и ключи создаются для стенда, не поставляются в skeleton.
|
||||
|
||||
**Архитектура:** Клиент — TLS → LB — mTLS → API; API/worker — TLS (+ клиентские cert где требуется матрицей) → зависимости; OpenBao выдаёт scoped credentials. Grafana/OTel остаются внутри доверенной сети.
|
||||
**Архитектура:** клиент — TLS → HAProxy — mTLS → API; API, worker, publisher, migrator и reconciler получают секреты через отдельные AppRole policies. Основные сервисы и observability сохраняют архитектуру lab6.
|
||||
|
||||
## Что сделать
|
||||
|
||||
@@ -50,33 +48,41 @@
|
||||
## Как начать и проверить
|
||||
|
||||
```sh
|
||||
# Работает прямо в skeleton, Python 3.9+; приложение не запускается:
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
cp .env.example .env
|
||||
.venv/bin/python scripts/bootstrap-secrets.py
|
||||
.venv/bin/python scripts/bootstrap-pki.py
|
||||
make check
|
||||
|
||||
# После реализации запустите свой сервер/стенд по REPORT.md.
|
||||
# По умолчанию тест использует https://localhost:8443.
|
||||
make test
|
||||
```
|
||||
# Сначала хранилища и OpenBao:
|
||||
docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao
|
||||
# Оператор инициализирует и unseal-ит OpenBao; root token/unseal shares
|
||||
# сохраняются вне репозитория и передаются только скрипту bootstrap:
|
||||
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init
|
||||
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal
|
||||
# Повторите unseal для требуемого порога shares.
|
||||
python3 scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token
|
||||
|
||||
`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость.
|
||||
|
||||
```sh
|
||||
# Compose-вариант после реализации:
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Дождитесь /health/ready; затем make test.
|
||||
# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md.
|
||||
make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-worker-isolation
|
||||
make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token
|
||||
```
|
||||
|
||||
Для HTTPS передайте публичный CA: `make test CA_FILE=/absolute/path/to/ca.crt`. CA certificate можно распространять; private keys создаются локально. `make test-security` проверяет только внешний TLS trust; внутренние отрицательные тесты реализуйте по [trust matrix](contracts/trust-matrix.md).
|
||||
`.env.example` содержит открытые локальные настройки. PKI и локальные bootstrap secrets создаются в игнорируемом Git каталоге `secrets/`. `compose.example.yaml` — worksheet; запускаемый стенд находится в `compose.yaml`. Образы закреплены версиями, данные приложения и OpenBao хранятся в named volumes. Обычная остановка сохраняет учебные данные (`docker compose down`); намеренное удаление данных выполняется отдельно (`docker compose down -v`).
|
||||
|
||||
|
||||
Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически):
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make test-primitives PYTHON=.venv/bin/python
|
||||
make test-openbao-acl PYTHON=.venv/bin/python
|
||||
make test-argon-budget
|
||||
.venv/bin/pip-audit -r requirements.txt
|
||||
```
|
||||
|
||||
Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте.
|
||||
|
||||
@@ -1,37 +1,110 @@
|
||||
# Отчёт — лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: в исходном отчёте lab6 не указаны.
|
||||
- Целевая оценка: 5; выбранный трек: 1 (компрометация worker). Ниже указаны повторяемая методика и результаты на локальном Compose-стенде.
|
||||
- Commit/PR: не создан.
|
||||
- Использованные источники, библиотеки и помощь ИИ: собственная реализация lab6; README/COURSE и контракты lab7; документация Argon2, Python `ssl`, OpenBao AppRole/ACL и Docker Compose. Использованы psycopg, boto3, Pillow, pika, OpenTelemetry и argon2-cffi (версии закреплены в `requirements.txt`). ИИ использовался для переноса, адаптации и проверки кода.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Compose v2, OpenSSL и Python 3.11+. Образы зафиксированы в `compose.yaml`; приложение собирается из `python:3.12.7-slim`. На машине разработки использован Python 3.13.2.
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
cp .env.example .env
|
||||
.venv/bin/python scripts/bootstrap-secrets.py
|
||||
.venv/bin/python scripts/bootstrap-pki.py
|
||||
docker compose --env-file .env config --quiet
|
||||
|
||||
# Поднять зависимости и OpenBao до приложения.
|
||||
docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao
|
||||
# Оператор выполняет init и unseal; shares и root token хранятся вне репозитория.
|
||||
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init
|
||||
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal
|
||||
# Повторить unseal до достижения порога, затем:
|
||||
.venv/bin/python scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token
|
||||
|
||||
docker compose --env-file .env up -d --build
|
||||
make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-primitives PYTHON=.venv/bin/python
|
||||
make test-internal-mtls
|
||||
make test-login-limit PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
```
|
||||
|
||||
Для остановки без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных и OpenBao state: `docker compose --env-file .env down -v`. Скрипты создают credentials и ключи только локально в исключённом из Git каталоге `secrets/`; не копируйте его в репозиторий. Секреты и unseal material не помещать в командную строку, отчёт или evidence.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Перенесены приложение, миграции, обработка вложений и тестовые сценарии lab6. PostgreSQL хранит пользователей, чаты, членство, сообщения, сессии, idempotency state и metadata вложений. SeaweedFS хранит оригиналы и variants; RabbitMQ обеспечивает at-least-once обработку через transactional outbox, отдельный publisher и worker; reconciler сверяет состояние. OTel Collector направляет сигналы в Prometheus, Loki и Tempo, Grafana загружает provisioning из репозитория.
|
||||
|
||||
HAProxy завершает внешний TLS на `localhost:8443`; сертификат ingress содержит SAN для `localhost` и `127.0.0.1`. HAProxy проверяет CA и имя API-сервера и предъявляет клиентский сертификат. API требует клиентский сертификат из локального CA и дополнительно разрешает только URI SAN `spiffe://mtusi2026lab7/load-balancer`. API listener не опубликован на host; internal health-check также использует TLS/mTLS.
|
||||
|
||||
Пароли записываются как Argon2id (`m=19456 KiB`, `t=2`, `p=1`); параметры hash проверяются до верификации и ограничены верхним пределом 64 MiB. Одновременно выполняется не более четырёх password hash операций в каждом API процессе; при исчерпании слотов запрос ограниченно ждёт 100 ms и получает 503/Retry-After, а не неограниченно аллоцирует память. Проверка legacy scrypt поддержана для миграции пользователей lab6; при успешном входе старый hash обновляется. Токены сессий продолжают храниться в PostgreSQL как SHA-256 hashes. Ограничение входа — пять попыток на username и шестьдесят на source IP за минуту; HAProxy заменяет `X-Real-IP`, поэтому значение от внешнего клиента не доверяется. Состояние общее для API-реплик, ответ блокировки — 429 с Retry-After.
|
||||
|
||||
OpenBao использует TLS, file storage в persistent volume и KV v2. Bootstrap создаёт отдельные AppRole и policy path для API, worker, publisher, migrator и reconciler. Приложения получают только собственные static KV values; операторский root token и unseal keys не передаются контейнерам. SecretID ограничен 24 часами, token TTL — 15 минут (max 1 час). Сертификат CA и private keys создаются скриптом на локальном стенде.
|
||||
|
||||
Фактические адреса приложения заданы в `compose.yaml`; `.env.example` — шаблон параметров локального запуска. Из переменных примера Compose использует порт ingress, число/лимиты API, session TTL, worker concurrency и telemetry sampling. Runtime secrets читаются через AppRole; в контейнерах нет root token. Python-сервисы работают как непривилегированный `app`, с `read_only`, tmpfs, `cap_drop: ALL` и `no-new-privileges`. Одноразовый `openbao-data-init` запускается от root только для установки владельца persistent volume и имеет единственную capability `CHOWN`; сам OpenBao drops all, оставляя `SETUID`/`SETGID`, нужные upstream entrypoint.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Структура и контракт | `make check` | PASS: OpenAPI structure, local refs, JSON и Python syntax | `scripts/check_contract.py` |
|
||||
| Схемы API | `make validate PYTHON=.venv/bin/python` | PASS: OpenAPI 3.1 и JSON Schema | `scripts/validate_openapi.py` |
|
||||
| Криптографические примитивы | `make test-primitives PYTHON=.venv/bin/python` | PASS: Argon2id, legacy scrypt verify/upgrade, URI SAN allowlist | `tests/test_security_primitives.py` |
|
||||
| Compose topology | `docker compose --env-file .env.example config --quiet` после генерации локальных секретов/PKI | PASS: Compose конфигурация разбирается | `compose.yaml` |
|
||||
| Локальная PKI | `openssl verify -CAfile secrets/pki/ca.crt ...` | PASS: ingress, API и LB client chain; client URI SAN содержит разрешённую identity | `scripts/bootstrap-pki.py` |
|
||||
| API smoke over HTTPS | `make test BASE_URL=https://localhost:8443 CA_FILE=...` | PASS на работающем Compose стенде | `tests/smoke.py` |
|
||||
| Внешняя проверка TLS trust | `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: доверенный TLS проходит, пустой trust store отвергается | `tests/security.py` |
|
||||
| Внутреннее mTLS | `make test-internal-mtls` | PASS: без cert, foreign CA и неверный URI SAN отклонены; разрешённая LB identity принята | `tests/internal_mtls.py`, `src/app.py`, `haproxy.cfg` |
|
||||
| Login rate limit | `make test-login-limit BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: шестая неуспешная попытка получает 429/Retry-After | `tests/login_rate_limit.py`, `migrations/006_login_rate_limits.sql` |
|
||||
| Внутренние TLS-соединения | Работающий Compose smoke, S3 attachment processing, worker broker consumer и проверки datasource из Grafana | PASS: PostgreSQL TLS, S3 HTTPS/mTLS, RabbitMQ AMQPS/mTLS, OTLP HTTPS/mTLS; из Grafana проверена CA-валидация Prometheus/Loki/Tempo | `compose.yaml`, `infra/postgres/`, `infra/otel/`, `src/object_store.py`, `src/jobs.py` |
|
||||
| OpenBao AppRole | `make test-openbao-acl PYTHON=.venv/bin/python` | PASS: пять сервисов читают собственные KV paths, cross-service read получает 403; token TTL ограничен | `tests/openbao_acl.py`, `scripts/bootstrap-openbao.py`, `infra/openbao/` |
|
||||
| Worker least privilege | `make test-worker-isolation` | PASS: DB identity `messenger_worker`; чтение `users`, S3 List и чтение originals/variants запрещены; нужные staging/originals/variants операции разрешены; RabbitMQ publish отклонён с 403 | `tests/worker_isolation.py`, `evidence/worker-isolation.json` |
|
||||
| Track 1: компрометация и recovery | `make test-worker-rotation ROOT_TOKEN_FILE=...` | PASS: старая AppRole SecretID отозвана и отвергнута; новая identity аутентифицируется; задача `queued → ready`, SHA-256 original и message link сохранены | `tests/worker_identity_rotation.py`, `evidence/worker-rotation.json` |
|
||||
| Ротация LB leaf-сертификата | `make test-leaf-rotation` | PASS: старый сертификат отвергнут, новый принят, HTTPS ingress healthy, Docker image IDs не изменились | `tests/leaf_certificate_rotation.py`, `evidence/leaf-rotation.json` |
|
||||
| Redaction telemetry | `make test-telemetry-redaction BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: 6 синтетических значений отсутствуют в 11 фактических Loki entries и 1 Tempo trace | `tests/telemetry_redaction.py`, `evidence/telemetry-redaction.json` |
|
||||
| Сохранность при restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: подтверждённое сообщение, attachment content и существующая сессия сохранены | `tests/resilience.py`, `evidence/persistence.json` |
|
||||
| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: две реплики до отказа; 108 успешных записей, 0 ошибок; первое восстановление за 3.463 s | `tests/resilience.py`, `evidence/failover.json` |
|
||||
| Argon2 resource budget | `make test-argon-budget` на API с лимитом 1 CPU/512 MiB | 20 verify операций: p50 23.007 ms, p95 28.288 ms, max 32.157 ms; из 12 одновременных запросов 6 выполнились и 6 получили capacity rejection, peak 4 активных hash операции, max RSS 162220 KiB (Linux ru_maxrss) | `tests/argon2_budget.py`, `evidence/argon2-budget.json` |
|
||||
| Python dependency scan | `.venv/bin/pip-audit -r requirements.txt` | PASS: 39 runtime packages, 0 известных уязвимостей в момент сканирования | `requirements.txt`, `evidence/dependency-audit.json` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Трек 1 — последствия компрометации worker
|
||||
|
||||
**Гипотеза.** Получив worker identity, атакующий может читать/писать staging bucket, писать originals/variants и потреблять process queue. Он не может читать пользовательские записи, перечислять buckets или читать originals/variants, публиковать свои задачи либо читать чужие OpenBao paths. После отзыва SecretID и его замены ожидающая задача должна завершиться без потери вложения или связи с сообщением.
|
||||
|
||||
**Методика.** На работающем Compose стенде выполнены `make test-worker-isolation` и `make test-worker-rotation ROOT_TOKEN_FILE=<operator-only-file>`. Первый сценарий проверяет реальные PostgreSQL grants, SeaweedFS `AccessDenied`, staging read/write, запись originals/variants, очистку временных объектов и broker ACL `403`. Второй останавливает worker, создаёт attachment и message link, выдаёт новую AppRole SecretID, отзывает прежнюю, проверяет отказ старого credential, пересоздаёт только worker и ждёт окончания обработки через публичный API. Root token и SecretID в вывод не включаются.
|
||||
|
||||
**Результат и сравнение.** В исходной общей схеме worker наследовал права общих application credentials; после разделения его DB role и S3/RabbitMQ identity имеют отдельные allowlists. В текущем измерении worker прочитал attachments, но получил отказ на `users`; S3 write для originals/variants и staging read/write прошли, а List и чтение originals/variants завершились `AccessDenied`; RabbitMQ отклонил публикацию с 403. Прежняя SecretID не прошла AppRole login, новая identity получила только собственный KV path. Attachment перешёл из `queued` в `ready`; SHA-256 исходных байтов и ссылка на attachment в сообщении сохранились. Машинные результаты сохранены в `evidence/worker-isolation.json` и `evidence/worker-rotation.json`.
|
||||
|
||||
**Границы эксперимента.** Проверялись заданные операции одной worker identity и один queued recovery сценарий на локальном Compose стенде; это не нагрузочный тест и не модель компрометации Docker host.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Проверены сохранность при restart, отказ одной API-реплики и восстановление записи; результаты приведены в таблице и JSON evidence. Отдельно измерялась стоимость Argon2 verify в ограниченном API container. Повторяемый профиль для учебного стенда:
|
||||
|
||||
```sh
|
||||
make load PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 \
|
||||
CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh \
|
||||
BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh \
|
||||
BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Нет клиентского сертификата / foreign CA / неверный URI SAN | API TLS listener отвергает соединение; LB identity проходит | Runtime тест: все три недоверенных запроса отклонены, LB identity принята | TLS-handshake |
|
||||
| Недоверенный CA внешнего endpoint | Проверка сертификата завершается TLS verification error | HTTPS smoke и проверка trust store выполнены | TLS-handshake |
|
||||
| Пять неуспешных login попыток в пределах минуты | Следующая попытка получает 429 и Retry-After | Runtime тест: шестая попытка отклонена с 429/Retry-After | Окно лимита — 60 секунд |
|
||||
| Отзыв worker SecretID при ожидающей задаче | Старое credential отклоняется, новая identity продолжает обработку | Runtime тест: old login denied; attachment `queued → ready`, содержимое и message link сохранены | Проверен один recovery цикл |
|
||||
| Restart API и stores | Сохранение подтверждённых данных, attachment content и существующей сессии | Runtime test: все данные доступны после restart | Проверка готовности после возврата сервисов |
|
||||
| Отказ одной API-реплики | Успешные записи через оставшуюся реплику, без дублей и потери подтверждённых данных | 108 успешных записей, 0 ошибок; 2 реплики до отказа | Первое восстановление за 3.463 s |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
`tests/load.py` — closed-loop генератор; его успешный HTTP throughput не является точным числом DB commits и не компенсирует coordinated omission. Измерения необходимо выполнять на одном и том же составе стенда с зафиксированными concurrency, длительностью, прогревом, аппаратными ресурсами и размером данных; сохранять вывод в `evidence/` без credentials и токенов.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Дополнительные измерения
|
||||
|
||||
Ротация LB leaf-сертификата проверена без пересборки образов: прежний сертификат отклоняется, новый проходит ingress health check. Redaction проверен на фактических записях Loki и трассе Tempo после генерации синтетических credentials и сообщения. Фактические протоколы и identities каждой связи зафиксированы в [матрице доверия](contracts/trust-matrix.md); остальные проверки и команды воспроизведения приведены в [tests/README.md](tests/README.md).
|
||||
|
||||
+871
@@ -0,0 +1,871 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
- postgres_tls_cert
|
||||
- ca_crt
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- ssl=on
|
||||
- -c
|
||||
- ssl_cert_file=/tls/server.crt
|
||||
- -c
|
||||
- ssl_key_file=/tls/server.key
|
||||
- -c
|
||||
- ssl_min_protocol_version=TLSv1.2
|
||||
- -c
|
||||
- hba_file=/etc/postgresql/pg_hba.conf
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- postgres_tls:/tls
|
||||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
depends_on:
|
||||
postgres-tls-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-management-alpine
|
||||
secrets:
|
||||
- source: rabbitmq_config
|
||||
target: /etc/rabbitmq/conf.d/10-defaults.conf
|
||||
- ca_crt
|
||||
- rabbitmq_tls_cert
|
||||
- rabbitmq_tls_key
|
||||
- rabbitmq_definitions
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
s3-core:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -filer=true
|
||||
- -s3=false
|
||||
- -volume.max=21
|
||||
volumes:
|
||||
- seaweed_data:/data
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
networks:
|
||||
- seaweed_internal
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- s3
|
||||
- -filer=s3-core:8888
|
||||
- -config=/run/secrets/s3_config
|
||||
- -iam.config=/run/secrets/s3_iam_config
|
||||
- -port=0
|
||||
- -port.https=8334
|
||||
- -cert.file=/run/secrets/s3_tls_cert
|
||||
- -key.file=/run/secrets/s3_tls_key
|
||||
- -cacert.file=/run/secrets/ca_crt
|
||||
- -tlsVerifyClientCert=true
|
||||
- -port.grpc=0
|
||||
- -port.iceberg=0
|
||||
- -port.lance=0
|
||||
- -concurrentFileUploadLimit=10
|
||||
- -concurrentUploadLimitMB=100
|
||||
- -readerCacheSizeMB=64
|
||||
- -autoCreateBucket=false
|
||||
- -ip.bind=0.0.0.0
|
||||
secrets:
|
||||
- s3_config
|
||||
- s3_iam_config
|
||||
- s3_tls_cert
|
||||
- s3_tls_key
|
||||
- ca_crt
|
||||
- s3_healthcheck_client_cert
|
||||
- s3_healthcheck_client_key
|
||||
depends_on:
|
||||
s3-core:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- default
|
||||
- seaweed_internal
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "code=$$(curl --cacert /run/secrets/ca_crt --cert /run/secrets/s3_healthcheck_client_cert --key /run/secrets/s3_healthcheck_client_key -sS -o /dev/null -w '%{http_code}' https://s3:8334/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/bootstrap_s3.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_bootstrap_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_bootstrap_client_key
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets:
|
||||
- ca_crt
|
||||
- s3_bootstrap_client_cert
|
||||
- s3_bootstrap_client_key
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
postgres-tls-init:
|
||||
image: alpine:3.21.3
|
||||
user: "0:0"
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- "cp /run/secrets/postgres_tls_cert /tls/server.crt && cp /run/secrets/postgres_tls_key /tls/server.key && chmod 0644 /tls/server.crt && chmod 0600 /tls/server.key && chown 70:70 /tls/server.crt /tls/server.key"
|
||||
secrets:
|
||||
- postgres_tls_cert
|
||||
- postgres_tls_key
|
||||
volumes:
|
||||
- postgres_tls:/tls
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=8m
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- CHOWN
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/migrate.py"]
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /run/secrets/ca_crt
|
||||
BAO_ROLE_ID_FILE: /run/secrets/migrate_role_id
|
||||
BAO_SECRET_ID_FILE: /run/secrets/migrate_secret_id
|
||||
BAO_SECRET_PATH: messenger/migrate
|
||||
WORKER_DB_PASSWORD_FILE: /run/secrets/worker_db_password
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_migrate_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_migrate_client_key
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
secrets:
|
||||
- ca_crt
|
||||
- migrate_role_id
|
||||
- migrate_secret_id
|
||||
- worker_db_password
|
||||
- s3_migrate_client_cert
|
||||
- s3_migrate_client_key
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
TLS_CERT_FILE: /run/secrets/api_tls_cert
|
||||
TLS_KEY_FILE: /run/secrets/api_tls_key
|
||||
TLS_CA_FILE: /run/secrets/ca_crt
|
||||
TLS_CLIENT_CERT_FILE: /run/secrets/lb_tls_cert
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /run/secrets/ca_crt
|
||||
BAO_ROLE_ID_FILE: /run/secrets/api_role_id
|
||||
BAO_SECRET_ID_FILE: /run/secrets/api_secret_id
|
||||
BAO_SECRET_PATH: messenger/api
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_api_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_api_client_key
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
ARGON2_MEMORY_KIB: "19456"
|
||||
ARGON2_ITERATIONS: "2"
|
||||
ARGON2_PARALLELISM: "1"
|
||||
PASSWORD_HASH_CONCURRENCY: "4"
|
||||
PASSWORD_HASH_WAIT_SECONDS: "0.1"
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX_SIZE: "8"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
DB_LOCK_TIMEOUT_MS: "2000"
|
||||
APP_ENV: lab
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_api_client_cert
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_api_client_key
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
secrets:
|
||||
- ca_crt
|
||||
- api_tls_cert
|
||||
- api_tls_key
|
||||
- lb_tls_cert
|
||||
- lb_tls_key
|
||||
- wrong_client_cert
|
||||
- wrong_client_key
|
||||
- foreign_client_cert
|
||||
- foreign_client_key
|
||||
- api_role_id
|
||||
- api_secret_id
|
||||
- otel_api_client_cert
|
||||
- otel_api_client_key
|
||||
- s3_api_client_cert
|
||||
- s3_api_client_key
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=128m
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- "import ssl,urllib.request; c=ssl.create_default_context(cafile='/run/secrets/ca_crt'); c.load_cert_chain('/run/secrets/lb_tls_cert','/run/secrets/lb_tls_key'); urllib.request.urlopen('https://127.0.0.1:8080/health/ready', context=c, timeout=2).read()"
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_LIMIT:-1.0}"
|
||||
memory: "${API_MEMORY_LIMIT:-512M}"
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
publisher:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "-m", "src.publisher"]
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /run/secrets/ca_crt
|
||||
BAO_ROLE_ID_FILE: /run/secrets/publisher_role_id
|
||||
BAO_SECRET_ID_FILE: /run/secrets/publisher_secret_id
|
||||
BAO_SECRET_PATH: messenger/publisher
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_publisher_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_publisher_client_key
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
APP_ENV: lab
|
||||
OTEL_SERVICE_NAME: messenger-publisher
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_publisher_client_cert
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_publisher_client_key
|
||||
BROKER_CA_FILE: /run/secrets/ca_crt
|
||||
BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_publisher_client_cert
|
||||
BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_publisher_client_key
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
secrets:
|
||||
- ca_crt
|
||||
- publisher_role_id
|
||||
- publisher_secret_id
|
||||
- otel_publisher_client_cert
|
||||
- otel_publisher_client_key
|
||||
- rabbitmq_publisher_client_cert
|
||||
- rabbitmq_publisher_client_key
|
||||
- s3_publisher_client_cert
|
||||
- s3_publisher_client_key
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "-m", "src.worker"]
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /run/secrets/ca_crt
|
||||
BAO_ROLE_ID_FILE: /run/secrets/worker_role_id
|
||||
BAO_SECRET_ID_FILE: /run/secrets/worker_secret_id
|
||||
BAO_SECRET_PATH: messenger/worker
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_worker_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_worker_client_key
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
APP_ENV: lab
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_worker_client_cert
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_worker_client_key
|
||||
BROKER_CA_FILE: /run/secrets/ca_crt
|
||||
BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_worker_client_cert
|
||||
BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_worker_client_key
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
secrets:
|
||||
- ca_crt
|
||||
- worker_role_id
|
||||
- worker_secret_id
|
||||
- otel_worker_client_cert
|
||||
- otel_worker_client_key
|
||||
- rabbitmq_worker_client_cert
|
||||
- rabbitmq_worker_client_key
|
||||
- s3_worker_client_cert
|
||||
- s3_worker_client_key
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=128m
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
reconciler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["sh", "scripts/reconcile-loop.sh"]
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /run/secrets/ca_crt
|
||||
BAO_ROLE_ID_FILE: /run/secrets/reconciler_role_id
|
||||
BAO_SECRET_ID_FILE: /run/secrets/reconciler_secret_id
|
||||
BAO_SECRET_PATH: messenger/reconciler
|
||||
S3_ENDPOINT: https://s3:8334
|
||||
S3_CA_FILE: /run/secrets/ca_crt
|
||||
S3_CLIENT_CERT_FILE: /run/secrets/s3_reconciler_client_cert
|
||||
S3_CLIENT_KEY_FILE: /run/secrets/s3_reconciler_client_key
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_STAGING_BUCKET: messenger-staging
|
||||
S3_VARIANTS_BUCKET: messenger-variants
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
|
||||
secrets:
|
||||
- ca_crt
|
||||
- reconciler_role_id
|
||||
- reconciler_secret_id
|
||||
- s3_reconciler_client_cert
|
||||
- s3_reconciler_client_key
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=32m
|
||||
cap_drop: [ALL]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
openbao:
|
||||
condition: service_started
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.163.2
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
secrets:
|
||||
- ca_crt
|
||||
- otel_collector_tls_cert
|
||||
- otel_collector_tls_key
|
||||
- tempo_tls_cert
|
||||
- loki_tls_cert
|
||||
depends_on:
|
||||
- tempo
|
||||
- loki
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --web.config.file=/etc/prometheus/web.yml
|
||||
secrets:
|
||||
- prometheus_tls_cert
|
||||
- prometheus_tls_key
|
||||
- ca_crt
|
||||
- collector_prometheus_client_cert
|
||||
- collector_prometheus_client_key
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/prometheus-web.yml:/etc/prometheus/web.yml:ro
|
||||
- ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
depends_on:
|
||||
- otel-collector
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.3.2
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
secrets:
|
||||
- loki_tls_cert
|
||||
- loki_tls_key
|
||||
volumes:
|
||||
- ./infra/otel/loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki_data:/loki
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
secrets:
|
||||
- tempo_tls_cert
|
||||
- tempo_tls_key
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo_data:/var/tempo
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.4.0
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
SSL_CERT_FILE: /run/secrets/ca_crt
|
||||
secrets:
|
||||
- grafana_admin_password
|
||||
- ca_crt
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.0.7-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8443}:8443"
|
||||
volumes:
|
||||
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
secrets:
|
||||
- ingress_pem
|
||||
- ca_crt
|
||||
- lb_client_pem
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
openbao:
|
||||
image: quay.io/openbao/openbao:2.1.1
|
||||
command: ["server", "-config=/bao/config.hcl"]
|
||||
ports:
|
||||
- "127.0.0.1:8200:8200"
|
||||
volumes:
|
||||
- openbao_data:/openbao/data
|
||||
- ./infra/openbao/config.hcl:/bao/config.hcl:ro
|
||||
secrets:
|
||||
- openbao_tls_cert
|
||||
- openbao_tls_key
|
||||
- ca_crt
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- SETGID
|
||||
- SETUID
|
||||
depends_on:
|
||||
openbao-data-init:
|
||||
condition: service_completed_successfully
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
openbao-data-init:
|
||||
image: alpine:3.21.3
|
||||
user: "0:0"
|
||||
command: ["sh", "-c", "mkdir -p /openbao/data && chown 100:1000 /openbao/data"]
|
||||
volumes:
|
||||
- openbao_data:/openbao/data
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=8m
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- CHOWN
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
restart: "no"
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
worker_db_password:
|
||||
file: ./secrets/worker_db_password
|
||||
postgres_tls_cert:
|
||||
file: ./secrets/pki/postgres.crt
|
||||
postgres_tls_key:
|
||||
file: ./secrets/pki/postgres.key
|
||||
s3_bootstrap_client_cert:
|
||||
file: ./secrets/pki/s3-bootstrap-client.crt
|
||||
s3_bootstrap_client_key:
|
||||
file: ./secrets/pki/s3-bootstrap-client.key
|
||||
s3_healthcheck_client_cert:
|
||||
file: ./secrets/pki/s3-healthcheck-client.crt
|
||||
s3_healthcheck_client_key:
|
||||
file: ./secrets/pki/s3-healthcheck-client.key
|
||||
s3_api_client_cert:
|
||||
file: ./secrets/pki/s3-api-client.crt
|
||||
s3_api_client_key:
|
||||
file: ./secrets/pki/s3-api-client.key
|
||||
s3_worker_client_cert:
|
||||
file: ./secrets/pki/s3-worker-client.crt
|
||||
s3_worker_client_key:
|
||||
file: ./secrets/pki/s3-worker-client.key
|
||||
s3_publisher_client_cert:
|
||||
file: ./secrets/pki/s3-publisher-client.crt
|
||||
s3_publisher_client_key:
|
||||
file: ./secrets/pki/s3-publisher-client.key
|
||||
s3_migrate_client_cert:
|
||||
file: ./secrets/pki/s3-migrate-client.crt
|
||||
s3_migrate_client_key:
|
||||
file: ./secrets/pki/s3-migrate-client.key
|
||||
s3_reconciler_client_cert:
|
||||
file: ./secrets/pki/s3-reconciler-client.crt
|
||||
s3_reconciler_client_key:
|
||||
file: ./secrets/pki/s3-reconciler-client.key
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_config:
|
||||
file: ./secrets/s3.json
|
||||
s3_iam_config:
|
||||
file: ./secrets/s3_iam.json
|
||||
s3_tls_cert:
|
||||
file: ./secrets/pki/s3.crt
|
||||
s3_tls_key:
|
||||
file: ./secrets/pki/s3.key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password
|
||||
rabbitmq_config:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
rabbitmq_definitions:
|
||||
file: ./secrets/rabbitmq_definitions.json
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
broker_worker_url:
|
||||
file: ./secrets/broker_worker_url
|
||||
broker_publisher_url:
|
||||
file: ./secrets/broker_publisher_url
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
ca_crt:
|
||||
file: ./secrets/pki/ca.crt
|
||||
api_tls_cert:
|
||||
file: ./secrets/pki/api.crt
|
||||
api_tls_key:
|
||||
file: ./secrets/pki/api.key
|
||||
lb_tls_cert:
|
||||
file: ./secrets/pki/lb-client.crt
|
||||
lb_tls_key:
|
||||
file: ./secrets/pki/lb-client.key
|
||||
lb_client_pem:
|
||||
file: ./secrets/pki/lb-client.pem
|
||||
ingress_pem:
|
||||
file: ./secrets/pki/ingress.pem
|
||||
openbao_tls_cert:
|
||||
file: ./secrets/pki/openbao.crt
|
||||
openbao_tls_key:
|
||||
file: ./secrets/pki/openbao.key
|
||||
rabbitmq_tls_cert:
|
||||
file: ./secrets/pki/rabbitmq.crt
|
||||
rabbitmq_tls_key:
|
||||
file: ./secrets/pki/rabbitmq.key
|
||||
otel_collector_tls_cert:
|
||||
file: ./secrets/pki/otel-collector.crt
|
||||
otel_collector_tls_key:
|
||||
file: ./secrets/pki/otel-collector.key
|
||||
tempo_tls_cert:
|
||||
file: ./secrets/pki/tempo.crt
|
||||
tempo_tls_key:
|
||||
file: ./secrets/pki/tempo.key
|
||||
loki_tls_cert:
|
||||
file: ./secrets/pki/loki.crt
|
||||
loki_tls_key:
|
||||
file: ./secrets/pki/loki.key
|
||||
prometheus_tls_cert:
|
||||
file: ./secrets/pki/prometheus.crt
|
||||
prometheus_tls_key:
|
||||
file: ./secrets/pki/prometheus.key
|
||||
collector_prometheus_client_cert:
|
||||
file: ./secrets/pki/collector-prometheus-client.crt
|
||||
collector_prometheus_client_key:
|
||||
file: ./secrets/pki/collector-prometheus-client.key
|
||||
otel_api_client_cert:
|
||||
file: ./secrets/pki/otel-api-client.crt
|
||||
otel_api_client_key:
|
||||
file: ./secrets/pki/otel-api-client.key
|
||||
otel_worker_client_cert:
|
||||
file: ./secrets/pki/otel-worker-client.crt
|
||||
otel_worker_client_key:
|
||||
file: ./secrets/pki/otel-worker-client.key
|
||||
otel_publisher_client_cert:
|
||||
file: ./secrets/pki/otel-publisher-client.crt
|
||||
otel_publisher_client_key:
|
||||
file: ./secrets/pki/otel-publisher-client.key
|
||||
rabbitmq_api_client_cert:
|
||||
file: ./secrets/pki/rabbitmq-api-client.crt
|
||||
rabbitmq_api_client_key:
|
||||
file: ./secrets/pki/rabbitmq-api-client.key
|
||||
rabbitmq_worker_client_cert:
|
||||
file: ./secrets/pki/rabbitmq-worker-client.crt
|
||||
rabbitmq_worker_client_key:
|
||||
file: ./secrets/pki/rabbitmq-worker-client.key
|
||||
rabbitmq_publisher_client_cert:
|
||||
file: ./secrets/pki/rabbitmq-publisher-client.crt
|
||||
rabbitmq_publisher_client_key:
|
||||
file: ./secrets/pki/rabbitmq-publisher-client.key
|
||||
wrong_client_cert:
|
||||
file: ./secrets/pki/wrong-client.crt
|
||||
wrong_client_key:
|
||||
file: ./secrets/pki/wrong-client.key
|
||||
foreign_client_cert:
|
||||
file: ./secrets/pki/foreign-client.crt
|
||||
foreign_client_key:
|
||||
file: ./secrets/pki/foreign-client.key
|
||||
api_role_id:
|
||||
file: ./secrets/openbao/api_role_id
|
||||
api_secret_id:
|
||||
file: ./secrets/openbao/api_secret_id
|
||||
worker_role_id:
|
||||
file: ./secrets/openbao/worker_role_id
|
||||
worker_secret_id:
|
||||
file: ./secrets/openbao/worker_secret_id
|
||||
publisher_role_id:
|
||||
file: ./secrets/openbao/publisher_role_id
|
||||
publisher_secret_id:
|
||||
file: ./secrets/openbao/publisher_secret_id
|
||||
migrate_role_id:
|
||||
file: ./secrets/openbao/migrate_role_id
|
||||
migrate_secret_id:
|
||||
file: ./secrets/openbao/migrate_secret_id
|
||||
reconciler_role_id:
|
||||
file: ./secrets/openbao/reconciler_role_id
|
||||
reconciler_secret_id:
|
||||
file: ./secrets/openbao/reconciler_secret_id
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
postgres_tls:
|
||||
seaweed_data:
|
||||
rabbitmq_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
openbao_data:
|
||||
|
||||
networks:
|
||||
seaweed_internal:
|
||||
internal: true
|
||||
+26
-28
@@ -1,37 +1,35 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия — lab7
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Для каждой связи указаны используемые endpoint, protocol, identity и фактический способ проверки. Состояние относится к реализации в `compose.yaml`.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| Связь | Фактический endpoint / protocol / identity | Проверка, срок действия | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Клиент → ingress/LB | `localhost:8443`, HTTPS; CA `secrets/pki/ca.crt`; ingress SAN `localhost`, `127.0.0.1`; внешний client certificate не требуется | Python HTTPS client проверяет CA и hostname; пользовательская идентичность — Bearer session; пять попыток на username и шестьдесят на source IP за минуту, затем 429/Retry-After | `haproxy.cfg`, `scripts/bootstrap-pki.py`, `tests/security.py`, `tests/login_rate_limit.py` |
|
||||
| LB → API (все реплики) | `api:8080`, TLS 1.2+, mTLS; CA `ca.crt`; HAProxy cert URI SAN `spiffe://mtusi2026lab7/load-balancer`; API SAN `DNS:api` | HAProxy проверяет CA и DNS hostname; API проверяет клиентскую CA chain и точный URI SAN; leaf lifetime 397 дней | `haproxy.cfg`, `src/app.py`, `src/security.py`, `tests/internal_mtls.py`; `make test-internal-mtls` |
|
||||
| API/worker → PostgreSQL | `postgres:5432`, PostgreSQL TLS over Compose network; CA verification (`sslmode=verify-full`); separate API and `messenger_worker` DB roles | Server certificate and hostname verified; worker role limited to attachment/outbox operations and columns | `compose.yaml`, `infra/postgres/pg_hba.conf`, `src/store.py`, `tests/worker_isolation.py` |
|
||||
| API/worker/publisher/reconciler → S3 | `s3:8334`, HTTPS with mTLS; separate client certificates and access keys; CA verification | API reads originals/variants and lists originals as configured; worker reads/writes staging and writes originals/variants, but cannot list or read originals/variants; publisher/migrator have no object actions | `compose.yaml`, `scripts/prepare-secure-secrets.py`, `src/object_store.py`, `tests/worker_isolation.py` |
|
||||
| Publisher/worker → broker | `rabbitmq:5671`, AMQPS with mTLS and CA/hostname verification; separate publisher and worker identities | Publisher can publish through the default exchange; worker consumes `attachments.process.v1` and is denied publishing; queue definitions are provisioned separately | `compose.yaml`, `src/jobs.py`, `scripts/prepare-secure-secrets.py`, `tests/worker_isolation.py` |
|
||||
| API/worker/publisher → Collector | `otel-collector:4318`, OTLP/HTTP over HTTPS with mTLS; distinct client certificates and CA verification | Collector requires a client certificate from the lab CA; application uses TLS exporter settings | `compose.yaml`, `infra/otel/collector.yaml`, `src/telemetry.py` |
|
||||
| Collector → telemetry backends | Prometheus scrape endpoint uses HTTPS/mTLS; Loki and Tempo use HTTPS with CA validation | Prometheus client certificate is checked by Collector; Collector validates backend CA | `infra/otel/collector.yaml`, `infra/otel/prometheus.yml`, `infra/otel/loki.yml`, `infra/otel/tempo.yml` |
|
||||
| Grafana → data sources | Prometheus, Loki and Tempo HTTPS service endpoints | Grafana uses the local CA trust bundle; endpoints were queried from the Grafana container | `compose.yaml`, `infra/otel/grafana/provisioning/datasources/datasources.yml` |
|
||||
| API/worker/agent → OpenBao | `https://openbao:8200`, CA verified; AppRole role_id + SecretID; KV v2 service-specific path | Per-service policy разрешает только `secret/data/messenger/<service>` read; SecretID TTL 24h, service token TTL 15m (max 1h) | `infra/openbao/config.hcl`, `scripts/bootstrap-openbao.py`, `src/config.py` |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | OpenBao опубликован только на `127.0.0.1:8200` по TLS; Grafana также на loopback; S3 admin используется bootstrap job внутри Compose network | OpenBao root token и unseal shares остаются у оператора; application services их не получают | `compose.yaml`, `infra/openbao/README.md`, `scripts/bootstrap-secrets.py` |
|
||||
| API → Redis/Valkey, если используется | N/A: session source of truth — PostgreSQL; Redis/Valkey не выбран | Не применяется | `src/store.py`, `compose.yaml` |
|
||||
| Replication/кластерные каналы, если добавлены | N/A: replicated database, broker cluster и multi-host transport не добавлены | Не применяется | `compose.yaml` |
|
||||
| LB/API health checks | LB health request использует mTLS backend connection; API self-check использует loopback HTTPS и разрешённый LB certificate | API listener не опубликован на host; health endpoint не обходит mTLS | `haproxy.cfg`, `compose.yaml`, `src/app.py` |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
## Отрицательные проверки
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
1. `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` выполняет положительный TLS probe и проверку отказа пустого trust store. Hostname проверяется клиентским TLS context.
|
||||
2. `make test-internal-mtls` проверяет API listener без client cert, с cert от foreign CA и с URI SAN другой службы; разрешённый LB cert должен пройти.
|
||||
3. `make test-login-limit` проверяет 429 и положительный Retry-After после неудачных попыток.
|
||||
4. `make test` проверяет ACL чат-участников, logout, сессии, сообщения и вложения; дополнительный `X-User-Id` не является источником identity после lab1.
|
||||
5. `make test-openbao-acl` проверяет, что каждая AppRole читает только собственный KV path и получает 403 на остальные service paths.
|
||||
6. `make test-worker-isolation` проверяет разрешённые и запрещённые операции worker в PostgreSQL, S3 и RabbitMQ.
|
||||
7. `make test-worker-rotation ROOT_TOKEN_FILE=...` отзывает старый worker SecretID, проверяет отказ старой identity и завершает queued attachment с заменённой identity.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
В Git хранятся только конфигурация и скрипты. CA private key, leaf keys, root token, unseal shares и AppRole SecretIDs создаются локально и хранятся в исключённом каталоге `secrets/`. Оператор отвечает за CA/unseal; OpenBao использует persistent file storage, TLS listener и AppRole auth. Root token не монтируется приложению. После restart оператор unseal-ит OpenBao при необходимости; приложение повторно аутентифицируется AppRole при старте.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
Redis и кластерные каналы не используются. После перезапуска OpenBao operator выполняет unseal; приложение не использует plaintext fallback при недоступном/sealed OpenBao.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"argon2_iterations": 2, "argon2_memory_kib": 19456, "argon2_parallelism": 1, "burst_capacity_rejections": 6, "burst_requests": 12, "burst_successes": 6, "hash_slots_per_process": 4, "hash_wait_seconds": 0.1, "max_rss_platform_units": 162220, "peak_active_hashes": 4, "platform": "linux", "sequential_samples": 20, "verify_latency_ms": {"max": 32.157, "p50": 23.007, "p95": 28.288}}
|
||||
@@ -0,0 +1 @@
|
||||
{"dependencies": [{"name": "psycopg", "version": "3.2.10", "vulns": []}, {"name": "psycopg-pool", "version": "3.3.3", "vulns": []}, {"name": "redis", "version": "5.2.1", "vulns": []}, {"name": "boto3", "version": "1.35.99", "vulns": []}, {"name": "pillow", "version": "13.0.0", "vulns": []}, {"name": "pika", "version": "1.3.2", "vulns": []}, {"name": "opentelemetry-api", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-sdk", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-http", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation-psycopg", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-botocore", "version": "0.66b2", "vulns": []}, {"name": "psutil", "version": "6.1.1", "vulns": []}, {"name": "argon2-cffi", "version": "23.1.0", "vulns": []}, {"name": "opentelemetry-exporter-http-transport", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-common", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-common", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-dbapi", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-proto", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-semantic-conventions", "version": "0.66b2", "vulns": []}, {"name": "psycopg-binary", "version": "3.2.10", "vulns": []}, {"name": "botocore", "version": "1.35.99", "vulns": []}, {"name": "googleapis-common-protos", "version": "1.76.0", "vulns": []}, {"name": "jmespath", "version": "1.1.0", "vulns": []}, {"name": "opentelemetry-propagator-aws-xray", "version": "1.0.2", "vulns": []}, {"name": "protobuf", "version": "7.36.2", "vulns": []}, {"name": "python-dateutil", "version": "2.9.0.post0", "vulns": []}, {"name": "requests", "version": "2.34.2", "vulns": []}, {"name": "charset-normalizer", "version": "3.5.2", "vulns": []}, {"name": "idna", "version": "3.20", "vulns": []}, {"name": "s3transfer", "version": "0.10.4", "vulns": []}, {"name": "urllib3", "version": "2.8.0", "vulns": []}, {"name": "wrapt", "version": "2.5.2", "vulns": []}, {"name": "certifi", "version": "2026.10.23", "vulns": []}, {"name": "six", "version": "1.17.0", "vulns": []}, {"name": "typing-extensions", "version": "4.16.0", "vulns": []}, {"name": "argon2-cffi-bindings", "version": "26.1.0", "vulns": []}, {"name": "cffi", "version": "2.1.1", "vulns": []}, {"name": "pycparser", "version": "3.12", "vulns": []}], "fixes": []}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"failed_attempts": 0,
|
||||
"first_recovery_seconds": 3.463,
|
||||
"observed_instances_before": 2,
|
||||
"successful_writes": 108,
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"https_ingress_healthy": true, "images_unchanged": true, "new_leaf_accepted": true, "new_leaf_sha256": "55e47bba381c871fa4c4dbdf34994526247be6429e37ae071199e73fa71d21ea", "old_leaf_rejected": true, "result": "PASS"}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"attachment_content_preserved": true,
|
||||
"confirmed_message_preserved": true,
|
||||
"existing_session_preserved": true,
|
||||
"hook": "scripts/restart.sh",
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"fixture": "tests/fixtures/sample.png",
|
||||
"invariants": {
|
||||
"ambiguous_confirm_republishes_stable_job": true,
|
||||
"confirmed_work_eventually_ready": true,
|
||||
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
|
||||
"duplicate_effects_create_no_second_variant_set": true,
|
||||
"stale_attempt_writes_no_effects": true,
|
||||
"stale_pipeline_cannot_publish_current_metadata": true
|
||||
},
|
||||
"result": "PASS",
|
||||
"scenarios": [
|
||||
{
|
||||
"deliveries": 2,
|
||||
"duplicate_delivery_writes": 0,
|
||||
"effective_object_writes": 4,
|
||||
"scenario": "duplicate_delivery_after_commit_before_ack"
|
||||
},
|
||||
{
|
||||
"attempts": 2,
|
||||
"scenario": "crash_after_claim",
|
||||
"terminal_status": "ready"
|
||||
},
|
||||
{
|
||||
"attempts": 2,
|
||||
"scenario": "crash_after_original_put",
|
||||
"terminal_status": "ready"
|
||||
},
|
||||
{
|
||||
"current_attempt": 2,
|
||||
"scenario": "out_of_order_attempt_delivery",
|
||||
"stale_attempt_extra_writes": 0
|
||||
},
|
||||
{
|
||||
"current_pipeline_version": 2,
|
||||
"published_variant_versions": [
|
||||
"v2"
|
||||
],
|
||||
"scenario": "stale_pipeline_after_newer_version"
|
||||
},
|
||||
{
|
||||
"outbox_recovered_after_restart": true,
|
||||
"published_after_restart": 1,
|
||||
"scenario": "publisher_crash_before_publish"
|
||||
},
|
||||
{
|
||||
"deliveries": 2,
|
||||
"same_message_id": true,
|
||||
"scenario": "publish_confirm_lost_after_broker_accept"
|
||||
},
|
||||
{
|
||||
"configured_timeout_seconds": 1,
|
||||
"observed_seconds": 1.007,
|
||||
"scenario": "transform_timeout_terminated"
|
||||
}
|
||||
],
|
||||
"track": "2"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"api_traces_scanned": 1, "leaks": 0, "log_entries_scanned": 11, "result": "PASS", "synthetic_values_checked": 6}
|
||||
@@ -0,0 +1 @@
|
||||
{"broker": {"broker_identity_connected": true, "publish_denied": true}, "database": {"attachments_read": true, "role": "messenger_worker", "users_read_denied": true}, "result": "PASS", "s3": {"messenger-attachments_list_denied": true, "messenger-attachments_read_denied": true, "messenger-attachments_write": true, "messenger-staging_list_denied": true, "messenger-staging_read": true, "messenger-staging_write": true, "messenger-variants_list_denied": true, "messenger-variants_read_denied": true, "messenger-variants_write": true, "temporary_objects_removed": true}}
|
||||
@@ -0,0 +1 @@
|
||||
{"attachment_state": ["queued", "ready"], "message_attachment_link_preserved": true, "old_worker_secret_id_revoked": true, "original_sha256_preserved": true, "replacement_worker_secret_id_authenticated": true, "result": "PASS"}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option http-keep-alive
|
||||
timeout connect 2s
|
||||
timeout client 15s
|
||||
timeout server 15s
|
||||
timeout http-request 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
|
||||
frontend messenger
|
||||
bind :8443 ssl crt /run/secrets/ingress_pem
|
||||
http-request set-header X-Real-IP %[src]
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check connect ssl sni api
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
option redispatch
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-10 api:8080 check ssl verify required ca-file /run/secrets/ca_crt crt /run/secrets/lb_client_pem verifyhost api sni str(api) resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# Инфраструктура студента
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Конфигурации выбранных сервисов находятся здесь; основная топология — в корневом `compose.yaml`. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. Текущие OpenBao policies/bootstrap описаны в `openbao/`; OTel provisioning перенесён из lab6 в `otel/`.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
|
||||
|
||||
+11
-4
@@ -1,7 +1,14 @@
|
||||
# OpenBao — реализовать конфигурацию и политики
|
||||
# OpenBao service configuration and bootstrap
|
||||
|
||||
Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key.
|
||||
`config.hcl` configures a TLS listener and persistent file storage; Compose runs OpenBao outside dev mode. `bootstrap-openbao.py` provisions KV v2 secrets and per-process AppRoles after operator init/unseal.
|
||||
|
||||
Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения.
|
||||
The API, worker, publisher, migrator and reconciler use separate AppRole credentials and policies. Each policy can read only its own `secret/data/messenger/<service>` KV v2 path. Service tokens have a 15-minute TTL and a one-hour maximum TTL; AppRole SecretIDs expire after 24 hours. Application code never receives the operator root token.
|
||||
|
||||
Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md.
|
||||
Operator workflow:
|
||||
|
||||
1. Generate local keys and bootstrap secrets with the scripts documented in the root README.
|
||||
2. Start the `openbao` service, run `bao operator init`, and unseal it using the operator-held shares. Keep the initial root token outside the repository.
|
||||
3. Run `python3 scripts/bootstrap-openbao.py --root-token-file <operator-only-file>`. It writes non-root AppRole IDs/SecretIDs under ignored `secrets/openbao/`; values are not printed.
|
||||
4. Start application services. On a normal restart, the file backend retains OpenBao state and the operator unseals it if required. If a service SecretID expires, provision a replacement and restart that service.
|
||||
|
||||
The script provisions distinct paths and identities; secret values originate from local bootstrap files and are not committed. Root token and unseal shares must never be mounted into application containers.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
path "secret/data/messenger/api" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
ui = false
|
||||
api_addr = "https://openbao:8200"
|
||||
cluster_addr = "https://openbao:8201"
|
||||
|
||||
storage "file" {
|
||||
path = "/openbao/data"
|
||||
}
|
||||
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
cluster_address = "0.0.0.0:8201"
|
||||
tls_cert_file = "/run/secrets/openbao_tls_cert"
|
||||
tls_key_file = "/run/secrets/openbao_tls_key"
|
||||
tls_min_version = "tls12"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
path "secret/data/messenger/worker" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
interval: 10s
|
||||
rules:
|
||||
- record: messenger:api_5xx_ratio:rate5m
|
||||
expr: |
|
||||
(sum(rate(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0))
|
||||
/
|
||||
clamp_min(sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])), 1e-9)
|
||||
- record: messenger:api_availability:ratio7d
|
||||
expr: |
|
||||
1 -
|
||||
(sum(increase(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[7d])) or vector(0))
|
||||
/
|
||||
clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[7d])), 1)
|
||||
- alert: MessengerApiErrorBudgetBurn
|
||||
expr: messenger:api_5xx_ratio:rate5m > 0.01
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: API error budget burn exceeds the 1% short-window budget
|
||||
description: The rolling 5-minute business API 5xx ratio is above 1%.
|
||||
runbook: Check API traces for the failing route, then inspect correlated logs and DB/S3 spans.
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 60 and max(messenger_queue_depth) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest attachment job has waited for more than 60 seconds
|
||||
description: Inspect worker availability, RabbitMQ health, and the worker trace for the oldest job.
|
||||
runbook: Check queue depth and age, worker consuming logs, broker, then recover the worker and confirm resolved state.
|
||||
- alert: MessengerDatabasePoolSaturation
|
||||
expr: max(messenger_db_pool_requests_waiting) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Requests are waiting for a database connection
|
||||
description: Compare DB waiters with request traces and PostgreSQL health.
|
||||
runbook: Inspect DB spans and pool use; verify database health and connection limits.
|
||||
@@ -0,0 +1,80 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
tls:
|
||||
cert_file: /run/secrets/otel_collector_tls_cert
|
||||
key_file: /run/secrets/otel_collector_tls_key
|
||||
client_ca_file: /run/secrets/ca_crt
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /run/secrets/otel_collector_tls_cert
|
||||
key_file: /run/secrets/otel_collector_tls_key
|
||||
client_ca_file: /run/secrets/ca_crt
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 384
|
||||
spike_limit_mib: 96
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 512
|
||||
send_batch_max_size: 1024
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
tls:
|
||||
cert_file: /run/secrets/otel_collector_tls_cert
|
||||
key_file: /run/secrets/otel_collector_tls_key
|
||||
client_ca_file: /run/secrets/ca_crt
|
||||
send_timestamps: true
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlp_grpc/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
ca_file: /run/secrets/ca_crt
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlp_http/loki:
|
||||
endpoint: https://loki:3100/otlp
|
||||
tls:
|
||||
ca_file: /run/secrets/ca_crt
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_grpc/tempo]
|
||||
@@ -0,0 +1,174 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "grafana", "uid": "-- Grafana --"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": false,
|
||||
"graphTooltip": 1,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum by (http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_route}}", "refId": "A"}],
|
||||
"title": "API requests per second",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 12, "x": 12, "y": 0},
|
||||
"id": 2,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",http_route!~\"/health/.*\"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m])), 1e-9)", "legendFormat": "5xx / business requests", "refId": "A"}],
|
||||
"title": "Business API 5xx ratio (5m)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 7},
|
||||
"id": 3,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
],
|
||||
"title": "API latency percentiles from merged histogram buckets",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 7},
|
||||
"id": 4,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
],
|
||||
"title": "Worker job duration percentiles from histogram buckets",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 15},
|
||||
"id": 5,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[5m]))", "legendFormat": "{{outcome}}", "refId": "A"}],
|
||||
"title": "Jobs processed / failed per second",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 15},
|
||||
"id": 6,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_queue_depth)", "legendFormat": "queued attachments", "refId": "A"}],
|
||||
"title": "Durable attachment backlog",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 15},
|
||||
"id": 7,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest queued job", "refId": "A"}],
|
||||
"title": "Age of oldest queued attachment",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 22},
|
||||
"id": 8,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_db_pool_connections_used)", "legendFormat": "used", "refId": "A"}, {"expr": "max(messenger_db_pool_requests_waiting)", "legendFormat": "waiting", "refId": "B"}],
|
||||
"title": "Database pool use / waiters",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 22},
|
||||
"id": 9,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "busy / capacity", "refId": "A"}],
|
||||
"title": "Worker saturation",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 22},
|
||||
"id": 10,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_process_memory_usage)", "legendFormat": "RSS", "refId": "A"}],
|
||||
"title": "Process memory",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 29},
|
||||
"id": 11,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_process_cpu_utilization)", "legendFormat": "CPU / core", "refId": "A"}],
|
||||
"title": "Process CPU utilization",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 16, "x": 8, "y": 29},
|
||||
"id": 12,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachments_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95 time to ready", "refId": "A"}],
|
||||
"title": "Durable acceptance to attachment ready",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 36},
|
||||
"id": 13,
|
||||
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": true, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
|
||||
"targets": [{"expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", "refId": "A"}],
|
||||
"title": "Structured API / worker logs",
|
||||
"type": "logs"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0.9, "max": 1}, "overrides": []},
|
||||
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 44},
|
||||
"id": 14,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
|
||||
"targets": [{"expr": "messenger:api_availability:ratio7d", "legendFormat": "7-day availability; objective 99%", "refId": "A"}],
|
||||
"title": "API SLI: 7-day rolling availability (99% SLO)",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"tags": ["messenger", "observability"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger observability",
|
||||
"uid": "messenger-observability",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
editable: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,39 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: https://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: https://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":"([a-f0-9]+)"'
|
||||
url: "$${__value.raw}"
|
||||
datasourceUid: tempo
|
||||
editable: false
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: https://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: "-1h"
|
||||
spanEndTimeShift: "1h"
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
customQuery: true
|
||||
query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"'
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
editable: false
|
||||
@@ -0,0 +1,37 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
http_tls_config:
|
||||
cert_file: /run/secrets/loki_tls_cert
|
||||
key_file: /run/secrets/loki_tls_key
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
@@ -0,0 +1,3 @@
|
||||
tls_server_config:
|
||||
cert_file: /run/secrets/prometheus_tls_cert
|
||||
key_file: /run/secrets/prometheus_tls_key
|
||||
@@ -0,0 +1,20 @@
|
||||
global:
|
||||
scrape_interval: 10s
|
||||
evaluation_interval: 10s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/secrets/ca_crt
|
||||
cert_file: /run/secrets/collector_prometheus_client_cert
|
||||
key_file: /run/secrets/collector_prometheus_client_key
|
||||
server_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:9464"]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,36 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
http_tls_config:
|
||||
cert_file: /run/secrets/tempo_tls_cert
|
||||
key_file: /run/secrets/tempo_tls_key
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
tls:
|
||||
cert_file: /run/secrets/tempo_tls_cert
|
||||
key_file: /run/secrets/tempo_tls_key
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /run/secrets/tempo_tls_cert
|
||||
key_file: /run/secrets/tempo_tls_key
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
trace_idle_period: 10s
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
@@ -0,0 +1,3 @@
|
||||
local all all trust
|
||||
hostssl all all all scram-sha-256
|
||||
hostnossl all all all reject
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method varchar(8) NOT NULL,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
|
||||
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
|
||||
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
|
||||
@@ -0,0 +1,59 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
staging_key text NOT NULL UNIQUE,
|
||||
job_id uuid NOT NULL UNIQUE,
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
lease_until timestamptz,
|
||||
status varchar(16) NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'),
|
||||
error_code varchar(64),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, ordinal)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
CREATE TABLE attachment_outbox (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
attempt integer NOT NULL CHECK (attempt >= 1),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
UNIQUE (job_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_outbox_pending_idx
|
||||
ON attachment_outbox (available_at, created_at)
|
||||
WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE reconciliation_incidents (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
|
||||
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, incident_type)
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
ALTER TABLE attachment_outbox
|
||||
DROP CONSTRAINT attachment_outbox_job_id_fkey;
|
||||
|
||||
CREATE TABLE attachment_pipeline_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
prior_job_id uuid NOT NULL,
|
||||
job_id uuid NOT NULL,
|
||||
prior_version integer NOT NULL CHECK (prior_version >= 1),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
action varchar(16) NOT NULL CHECK (action = 'replay'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_pipeline_events_attachment_created_idx
|
||||
ON attachment_pipeline_events (attachment_id, created_at);
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_outbox
|
||||
ADD COLUMN traceparent varchar(55),
|
||||
ADD COLUMN tracestate varchar(512);
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE login_rate_limits (
|
||||
username_key bytea PRIMARY KEY,
|
||||
window_started_at timestamptz NOT NULL,
|
||||
attempts integer NOT NULL CHECK (attempts > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX login_rate_limits_window_idx ON login_rate_limits (window_started_at);
|
||||
@@ -1,2 +1,3 @@
|
||||
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
|
||||
# Development-only schema validation and dependency auditing tools.
|
||||
openapi-spec-validator==0.7.2
|
||||
pip-audit==2.10.1
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.3.3
|
||||
redis==5.2.1
|
||||
boto3==1.35.99
|
||||
Pillow==13.0.0
|
||||
pika==1.3.2
|
||||
opentelemetry-api==1.45.2
|
||||
opentelemetry-sdk==1.45.2
|
||||
opentelemetry-exporter-otlp-proto-http==1.45.2
|
||||
opentelemetry-instrumentation-psycopg==0.66b2
|
||||
opentelemetry-instrumentation-botocore==0.66b2
|
||||
psutil==6.1.1
|
||||
argon2-cffi==23.1.0
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
from typing import Optional
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets"
|
||||
ROLE_DIR = SECRET_DIR / "openbao"
|
||||
SERVICES = {
|
||||
"api": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
"worker": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
"publisher": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
"migrate": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
"reconciler": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Provision scoped OpenBao AppRoles for this local lab.")
|
||||
parser.add_argument("--root-token-file", required=True, help="Operator-only file outside version control")
|
||||
parser.add_argument("--ca-file", default=str(SECRET_DIR / "pki" / "ca.crt"))
|
||||
parser.add_argument("--address", default="https://localhost:8200")
|
||||
args = parser.parse_args()
|
||||
root_token = Path(args.root_token_file).read_text(encoding="utf-8").strip()
|
||||
if not root_token:
|
||||
raise SystemExit("Root token file is empty.")
|
||||
ROLE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
context = ssl.create_default_context(cafile=args.ca_file)
|
||||
|
||||
def request(method: str, path: str, payload: Optional[dict] = None) -> dict:
|
||||
headers = {"X-Vault-Token": root_token, "Content-Type": "application/json"}
|
||||
body = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
req = urllib.request.Request(
|
||||
args.address.rstrip("/") + "/v1/" + path,
|
||||
data=body,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=context, timeout=8) as response:
|
||||
content = response.read(1024 * 1024)
|
||||
return json.loads(content) if content else {}
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(f"OpenBao API request failed with HTTP {exc.code}") from None
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
||||
raise RuntimeError(f"OpenBao API request failed ({type(exc).__name__})") from None
|
||||
|
||||
mounts = request("GET", "sys/mounts")
|
||||
if "secret/" not in mounts.get("data", {}):
|
||||
request("POST", "sys/mounts/secret", {"type": "kv", "options": {"version": "2"}})
|
||||
auth = request("GET", "sys/auth")
|
||||
if "approle/" not in auth.get("data", {}):
|
||||
request("POST", "sys/auth/approle", {"type": "approle"})
|
||||
|
||||
local_values = {
|
||||
"DATABASE_URL": (SECRET_DIR / "database_url").read_text(encoding="utf-8").strip(),
|
||||
"WORKER_DATABASE_URL": (SECRET_DIR / "worker_database_url").read_text(encoding="utf-8").strip(),
|
||||
"BROKER_URL": "",
|
||||
"S3_ACCESS_KEY_ID": (SECRET_DIR / "s3_access_key_id").read_text(encoding="utf-8").strip(),
|
||||
"S3_SECRET_ACCESS_KEY": (SECRET_DIR / "s3_secret_access_key").read_text(encoding="utf-8").strip(),
|
||||
}
|
||||
service_s3_files = {
|
||||
"api": ("s3_access_key_id", "s3_secret_access_key"),
|
||||
**{
|
||||
service: (f"{service}_s3_access_key_id", f"{service}_s3_secret_access_key")
|
||||
for service in ("worker", "publisher", "migrate", "reconciler")
|
||||
},
|
||||
}
|
||||
|
||||
for service, keys in SERVICES.items():
|
||||
if service in ("worker", "publisher"):
|
||||
local_values["BROKER_URL"] = (
|
||||
SECRET_DIR / f"broker_{service}_url"
|
||||
).read_text(encoding="utf-8").strip()
|
||||
policy_name = f"messenger-{service}"
|
||||
policy = (
|
||||
f'path "secret/data/messenger/{service}" {{\n'
|
||||
' capabilities = ["read"]\n'
|
||||
'}\n'
|
||||
)
|
||||
request("PUT", f"sys/policies/acl/{policy_name}", {"policy": policy})
|
||||
request(
|
||||
"POST",
|
||||
f"auth/approle/role/{service}",
|
||||
{
|
||||
"token_policies": [policy_name],
|
||||
"token_ttl": "15m",
|
||||
"token_max_ttl": "1h",
|
||||
"secret_id_ttl": "24h",
|
||||
"secret_id_num_uses": 0,
|
||||
},
|
||||
)
|
||||
service_values = dict(local_values)
|
||||
access_key_file, secret_key_file = service_s3_files[service]
|
||||
service_values["S3_ACCESS_KEY_ID"] = (
|
||||
SECRET_DIR / access_key_file
|
||||
).read_text(encoding="utf-8").strip()
|
||||
service_values["S3_SECRET_ACCESS_KEY"] = (
|
||||
SECRET_DIR / secret_key_file
|
||||
).read_text(encoding="utf-8").strip()
|
||||
if service == "worker":
|
||||
service_values["DATABASE_URL"] = local_values["WORKER_DATABASE_URL"]
|
||||
request(
|
||||
"POST",
|
||||
f"secret/data/messenger/{service}",
|
||||
{"data": {key: service_values[key] for key in keys}},
|
||||
)
|
||||
role = request("GET", f"auth/approle/role/{service}/role-id")
|
||||
role_id = role.get("data", {}).get("role_id")
|
||||
secret = request("POST", f"auth/approle/role/{service}/secret-id", {})
|
||||
secret_id = secret.get("data", {}).get("secret_id")
|
||||
if not role_id or not secret_id:
|
||||
raise RuntimeError(f"OpenBao did not return credentials for the {service} role")
|
||||
for suffix, value in (("role_id", role_id), ("secret_id", secret_id)):
|
||||
path = ROLE_DIR / f"{service}_{suffix}"
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
|
||||
print("OpenBao policies, scoped AppRoles and service credentials provisioned; values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI_DIR = ROOT / "secrets" / "pki"
|
||||
|
||||
|
||||
def openssl(*args: str) -> None:
|
||||
subprocess.run(
|
||||
["openssl", *args],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def create_leaf(
|
||||
work: Path,
|
||||
name: str,
|
||||
subject: str,
|
||||
sans: str,
|
||||
extended_usage: str,
|
||||
ca_key: Path,
|
||||
ca_cert: Path,
|
||||
) -> tuple[Path, Path]:
|
||||
key = PKI_DIR / f"{name}.key"
|
||||
cert = PKI_DIR / f"{name}.crt"
|
||||
request = work / f"{name}.csr"
|
||||
extensions = work / f"{name}.ext"
|
||||
openssl(
|
||||
"req",
|
||||
"-new",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
str(key),
|
||||
"-out",
|
||||
str(request),
|
||||
"-subj",
|
||||
subject,
|
||||
)
|
||||
extensions.write_text(
|
||||
"basicConstraints=critical,CA:FALSE\n"
|
||||
"keyUsage=critical,digitalSignature,keyEncipherment\n"
|
||||
f"extendedKeyUsage={extended_usage}\n"
|
||||
f"subjectAltName={sans}\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
openssl(
|
||||
"x509",
|
||||
"-req",
|
||||
"-in",
|
||||
str(request),
|
||||
"-CA",
|
||||
str(ca_cert),
|
||||
"-CAkey",
|
||||
str(ca_key),
|
||||
"-CAcreateserial",
|
||||
"-out",
|
||||
str(cert),
|
||||
"-days",
|
||||
"397",
|
||||
"-sha256",
|
||||
"-extfile",
|
||||
str(extensions),
|
||||
)
|
||||
os.chmod(key, 0o600)
|
||||
os.chmod(cert, 0o644)
|
||||
return cert, key
|
||||
|
||||
|
||||
def bundle(cert: Path, key: Path, output: Path) -> None:
|
||||
output.write_bytes(cert.read_bytes() + key.read_bytes())
|
||||
os.chmod(output, 0o600)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if PKI_DIR.exists():
|
||||
raise SystemExit("PKI files already exist; rotate them with an explicit rollover procedure.")
|
||||
if not shutil.which("openssl"):
|
||||
raise SystemExit("OpenSSL is required to generate local certificates.")
|
||||
|
||||
PKI_DIR.mkdir(mode=0o700, parents=True)
|
||||
ca_key = PKI_DIR / "ca.key"
|
||||
ca_cert = PKI_DIR / "ca.crt"
|
||||
openssl(
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:3072",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
str(ca_key),
|
||||
"-out",
|
||||
str(ca_cert),
|
||||
"-days",
|
||||
"3650",
|
||||
"-sha256",
|
||||
"-subj",
|
||||
"/CN=mtusi2026lab7-local-ca",
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:TRUE",
|
||||
"-addext",
|
||||
"keyUsage=critical,keyCertSign,cRLSign",
|
||||
)
|
||||
os.chmod(ca_key, 0o600)
|
||||
os.chmod(ca_cert, 0o644)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="lab7-pki-") as directory:
|
||||
work = Path(directory)
|
||||
foreign_ca_key = PKI_DIR / "foreign-ca.key"
|
||||
foreign_ca_cert = PKI_DIR / "foreign-ca.crt"
|
||||
openssl(
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
str(foreign_ca_key),
|
||||
"-out",
|
||||
str(foreign_ca_cert),
|
||||
"-days",
|
||||
"365",
|
||||
"-sha256",
|
||||
"-subj",
|
||||
"/CN=mtusi2026lab7-untrusted-ca",
|
||||
"-addext",
|
||||
"basicConstraints=critical,CA:TRUE",
|
||||
"-addext",
|
||||
"keyUsage=critical,keyCertSign,cRLSign",
|
||||
)
|
||||
os.chmod(foreign_ca_key, 0o600)
|
||||
os.chmod(foreign_ca_cert, 0o644)
|
||||
ingress_cert, ingress_key = create_leaf(
|
||||
work,
|
||||
"ingress",
|
||||
"/CN=localhost",
|
||||
"DNS:localhost,IP:127.0.0.1",
|
||||
"serverAuth",
|
||||
ca_key,
|
||||
ca_cert,
|
||||
)
|
||||
create_leaf(
|
||||
work,
|
||||
"api",
|
||||
"/CN=api",
|
||||
"DNS:api,IP:127.0.0.1",
|
||||
"serverAuth",
|
||||
ca_key,
|
||||
ca_cert,
|
||||
)
|
||||
client_cert, client_key = create_leaf(
|
||||
work,
|
||||
"lb-client",
|
||||
"/CN=lab7-load-balancer",
|
||||
"URI:spiffe://mtusi2026lab7/load-balancer",
|
||||
"clientAuth",
|
||||
ca_key,
|
||||
ca_cert,
|
||||
)
|
||||
create_leaf(
|
||||
work,
|
||||
"wrong-client",
|
||||
"/CN=lab7-untrusted-service",
|
||||
"URI:spiffe://mtusi2026lab7/other-service",
|
||||
"clientAuth",
|
||||
ca_key,
|
||||
ca_cert,
|
||||
)
|
||||
create_leaf(
|
||||
work,
|
||||
"foreign-client",
|
||||
"/CN=lab7-foreign-service",
|
||||
"URI:spiffe://mtusi2026lab7/load-balancer",
|
||||
"clientAuth",
|
||||
foreign_ca_key,
|
||||
foreign_ca_cert,
|
||||
)
|
||||
create_leaf(
|
||||
work,
|
||||
"openbao",
|
||||
"/CN=openbao",
|
||||
"DNS:openbao,DNS:localhost,IP:127.0.0.1",
|
||||
"serverAuth",
|
||||
ca_key,
|
||||
ca_cert,
|
||||
)
|
||||
bundle(ingress_cert, ingress_key, PKI_DIR / "ingress.pem")
|
||||
bundle(client_cert, client_key, PKI_DIR / "lb-client.pem")
|
||||
print("Local CA and leaf certificates created under secrets/pki with private permissions.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets"
|
||||
|
||||
|
||||
def main():
|
||||
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
|
||||
os.chmod(SECRET_DIR, 0o700)
|
||||
names = (
|
||||
"postgres_password",
|
||||
"database_url",
|
||||
"worker_db_password",
|
||||
"worker_database_url",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
"worker_s3_access_key_id",
|
||||
"worker_s3_secret_access_key",
|
||||
"publisher_s3_access_key_id",
|
||||
"publisher_s3_secret_access_key",
|
||||
"migrate_s3_access_key_id",
|
||||
"migrate_s3_secret_access_key",
|
||||
"reconciler_s3_access_key_id",
|
||||
"reconciler_s3_secret_access_key",
|
||||
"s3_admin_access_key_id",
|
||||
"s3_admin_secret_access_key",
|
||||
"rabbitmq_password",
|
||||
"rabbitmq.conf",
|
||||
"rabbitmq_definitions.json",
|
||||
"broker_url",
|
||||
"broker_worker_url",
|
||||
"broker_publisher_url",
|
||||
"s3.json",
|
||||
"s3_iam.json",
|
||||
"grafana_admin_password",
|
||||
)
|
||||
existing = [name for name in names if (SECRET_DIR / name).exists()]
|
||||
if existing:
|
||||
raise SystemExit(
|
||||
"Secret files already exist; refusing to rotate credentials without a migration: "
|
||||
+ ", ".join(existing)
|
||||
)
|
||||
|
||||
db_password = secrets.token_hex(24)
|
||||
worker_db_password = secrets.token_urlsafe(32)
|
||||
broker_password = secrets.token_urlsafe(32)
|
||||
worker_broker_password = secrets.token_urlsafe(32)
|
||||
publisher_broker_password = secrets.token_urlsafe(32)
|
||||
values = {
|
||||
"postgres_password": db_password,
|
||||
"database_url": (
|
||||
f"postgresql://messenger:{db_password}@postgres:5432/messenger"
|
||||
"?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt"
|
||||
),
|
||||
"worker_db_password": worker_db_password,
|
||||
"worker_database_url": (
|
||||
f"postgresql://messenger_worker:{worker_db_password}@postgres:5432/messenger"
|
||||
"?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt"
|
||||
),
|
||||
"s3_access_key_id": "mngr" + secrets.token_hex(8),
|
||||
"s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"worker_s3_access_key_id": "wrk" + secrets.token_hex(8),
|
||||
"worker_s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"publisher_s3_access_key_id": "pub" + secrets.token_hex(8),
|
||||
"publisher_s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"migrate_s3_access_key_id": "mig" + secrets.token_hex(8),
|
||||
"migrate_s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"reconciler_s3_access_key_id": "rec" + secrets.token_hex(8),
|
||||
"reconciler_s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
|
||||
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
|
||||
"rabbitmq_password": broker_password,
|
||||
"broker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f",
|
||||
"broker_worker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f",
|
||||
"broker_publisher_url": f"amqps://publisher:{publisher_broker_password}@rabbitmq:5671/%2f",
|
||||
"grafana_admin_password": secrets.token_urlsafe(32),
|
||||
}
|
||||
for name, value in values.items():
|
||||
if name.startswith("broker_") and name != "broker_url":
|
||||
descriptor = os.open(SECRET_DIR / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
continue
|
||||
path = SECRET_DIR / name
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
descriptor = os.open(
|
||||
SECRET_DIR / "rabbitmq.conf",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write("default_user = messenger\n")
|
||||
output.write(f"default_pass = {broker_password}\n")
|
||||
output.write("listeners.tcp = none\n")
|
||||
output.write("listeners.ssl.default = 5671\n")
|
||||
output.write("ssl_options.cacertfile = /run/secrets/ca_crt\n")
|
||||
output.write("ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n")
|
||||
output.write("ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n")
|
||||
output.write("ssl_options.verify = verify_peer\n")
|
||||
output.write("ssl_options.fail_if_no_peer_cert = true\n")
|
||||
output.write("ssl_options.versions.1 = tlsv1.2\n")
|
||||
output.write("definitions.import_backend = local_filesystem\n")
|
||||
output.write("definitions.local.path = /run/secrets/rabbitmq_definitions.json\n")
|
||||
definitions = {
|
||||
"users": [],
|
||||
"vhosts": [{"name": "/"}],
|
||||
"permissions": [],
|
||||
"queues": [
|
||||
{
|
||||
"name": "attachments.process.dead",
|
||||
"vhost": "/",
|
||||
"durable": True,
|
||||
"auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
|
||||
},
|
||||
{
|
||||
"name": "attachments.process.v1",
|
||||
"vhost": "/",
|
||||
"durable": True,
|
||||
"auto_delete": False,
|
||||
"arguments": {
|
||||
"x-queue-type": "quorum",
|
||||
"x-quorum-initial-group-size": 1,
|
||||
"x-dead-letter-exchange": "",
|
||||
"x-dead-letter-routing-key": "attachments.process.dead",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
for username, password, configure, write, read in (
|
||||
("worker", worker_broker_password, r"^$", r"^$", r"^attachments\.process\.v1$"),
|
||||
("publisher", publisher_broker_password, r"^$", r"^amq\.default$", r"^$"),
|
||||
):
|
||||
salt = secrets.token_bytes(4)
|
||||
password_hash = __import__("hashlib").sha256(salt + password.encode("utf-8")).digest()
|
||||
definitions["users"].append({
|
||||
"name": username,
|
||||
"password_hash": __import__("base64").b64encode(salt + password_hash).decode("ascii"),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256",
|
||||
"tags": "",
|
||||
})
|
||||
definitions["permissions"].append({
|
||||
"user": username,
|
||||
"vhost": "/",
|
||||
"configure": configure,
|
||||
"write": write,
|
||||
"read": read,
|
||||
})
|
||||
descriptor = os.open(
|
||||
SECRET_DIR / "rabbitmq_definitions.json",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o600,
|
||||
)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
json.dump(definitions, output, separators=(",", ":"))
|
||||
output.write("\n")
|
||||
os.chmod(SECRET_DIR / "rabbitmq_definitions.json", 0o644)
|
||||
s3_config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "bootstrap-admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_admin_access_key_id"],
|
||||
"secretKey": values["s3_admin_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": ["Admin"],
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_access_key_id"],
|
||||
"secretKey": values["s3_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"List:messenger-attachments",
|
||||
"Read:messenger-variants",
|
||||
"Write:messenger-staging",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "messenger-worker",
|
||||
"credentials": [{
|
||||
"accessKey": values["worker_s3_access_key_id"],
|
||||
"secretKey": values["worker_s3_secret_access_key"],
|
||||
}],
|
||||
"actions": [
|
||||
"Read:messenger-staging",
|
||||
"Write:messenger-staging",
|
||||
"Write:messenger-attachments",
|
||||
"Write:messenger-variants",
|
||||
],
|
||||
},
|
||||
*[
|
||||
{
|
||||
"name": f"messenger-{service}",
|
||||
"credentials": [{
|
||||
"accessKey": values[f"{service}_s3_access_key_id"],
|
||||
"secretKey": values[f"{service}_s3_secret_access_key"],
|
||||
}],
|
||||
"actions": [],
|
||||
}
|
||||
for service in ("publisher", "migrate")
|
||||
],
|
||||
{
|
||||
"name": "messenger-reconciler",
|
||||
"credentials": [{
|
||||
"accessKey": values["reconciler_s3_access_key_id"],
|
||||
"secretKey": values["reconciler_s3_secret_access_key"],
|
||||
}],
|
||||
"actions": [
|
||||
f"{action}:{bucket}"
|
||||
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants")
|
||||
for action in ("Read", "Write", "List")
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
json.dump(s3_config, output, separators=(",", ":"))
|
||||
output.write("\n")
|
||||
iam_config = {
|
||||
"sts": {"signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")},
|
||||
"defaultEffect": "Deny",
|
||||
}
|
||||
descriptor = os.open(SECRET_DIR / "s3_iam.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
json.dump(iam_config, output, separators=(",", ":"))
|
||||
output.write("\n")
|
||||
role_dir = SECRET_DIR / "openbao"
|
||||
role_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
for service in ("api", "worker", "publisher", "migrate", "reconciler"):
|
||||
for suffix in ("role_id", "secret_id"):
|
||||
path = role_dir / f"{service}_{suffix}"
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
os.close(descriptor)
|
||||
print("Local secrets generated with owner-only permissions; values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI_DIR = ROOT / "secrets" / "pki"
|
||||
SERVICES = {
|
||||
"rabbitmq": ("DNS:rabbitmq", "serverAuth"),
|
||||
"s3": ("DNS:s3", "serverAuth"),
|
||||
"postgres": ("DNS:postgres", "serverAuth"),
|
||||
"otel-collector": ("DNS:otel-collector", "serverAuth"),
|
||||
"tempo": ("DNS:tempo", "serverAuth"),
|
||||
"loki": ("DNS:loki", "serverAuth"),
|
||||
"prometheus": ("DNS:prometheus", "serverAuth"),
|
||||
"collector-prometheus-client": ("URI:spiffe://mtusi2026lab7/collector", "clientAuth"),
|
||||
"otel-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
|
||||
"otel-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
|
||||
"otel-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
|
||||
"rabbitmq-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
|
||||
"rabbitmq-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
|
||||
"rabbitmq-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
|
||||
"s3-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
|
||||
"s3-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
|
||||
"s3-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
|
||||
"s3-migrate-client": ("URI:spiffe://mtusi2026lab7/migrate", "clientAuth"),
|
||||
"s3-reconciler-client": ("URI:spiffe://mtusi2026lab7/reconciler", "clientAuth"),
|
||||
"s3-bootstrap-client": ("URI:spiffe://mtusi2026lab7/s3-bootstrap", "clientAuth"),
|
||||
"s3-healthcheck-client": ("URI:spiffe://mtusi2026lab7/s3-healthcheck", "clientAuth"),
|
||||
}
|
||||
|
||||
|
||||
def openssl(*args: str) -> None:
|
||||
subprocess.run(
|
||||
["openssl", *args],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Issue local short-lived mTLS service identities.")
|
||||
parser.add_argument("--days", type=int, default=30)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.days <= 90:
|
||||
parser.error("service certificate lifetime must be between 1 and 90 days")
|
||||
if not (PKI_DIR / "ca.key").is_file() or not (PKI_DIR / "ca.crt").is_file():
|
||||
raise SystemExit("Create the local CA first with scripts/bootstrap-pki.py.")
|
||||
if not shutil.which("openssl"):
|
||||
raise SystemExit("OpenSSL is required to issue service certificates.")
|
||||
missing = {
|
||||
name: identity for name, identity in SERVICES.items()
|
||||
if not (PKI_DIR / f"{name}.crt").exists()
|
||||
}
|
||||
if not missing:
|
||||
raise SystemExit("All service certificates already exist; rotate them explicitly.")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="lab7-service-pki-") as directory:
|
||||
work = Path(directory)
|
||||
for name, (san, usage) in missing.items():
|
||||
key = PKI_DIR / f"{name}.key"
|
||||
cert = PKI_DIR / f"{name}.crt"
|
||||
csr = work / f"{name}.csr"
|
||||
extension = work / f"{name}.ext"
|
||||
openssl(
|
||||
"req", "-new", "-newkey", "rsa:2048", "-nodes",
|
||||
"-keyout", str(key), "-out", str(csr), "-subj", f"/CN={name}",
|
||||
)
|
||||
extension.write_text(
|
||||
"basicConstraints=critical,CA:FALSE\n"
|
||||
"keyUsage=critical,digitalSignature,keyEncipherment\n"
|
||||
f"extendedKeyUsage={usage}\n"
|
||||
f"subjectAltName={san}\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
openssl(
|
||||
"x509", "-req", "-in", str(csr), "-CA", str(PKI_DIR / "ca.crt"),
|
||||
"-CAkey", str(PKI_DIR / "ca.key"), "-CAcreateserial", "-out", str(cert),
|
||||
"-days", str(args.days), "-sha256", "-extfile", str(extension),
|
||||
)
|
||||
os.chmod(key, 0o600)
|
||||
os.chmod(cert, 0o644)
|
||||
print(f"Issued {len(missing)} service certificates with {args.days}-day validity.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets"
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (SECRET_DIR / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def write(name: str, content: str, replace: bool = False) -> None:
|
||||
path = SECRET_DIR / name
|
||||
flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if replace else os.O_EXCL)
|
||||
descriptor = os.open(path, flags, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(content)
|
||||
if not content.endswith("\n"):
|
||||
output.write("\n")
|
||||
|
||||
|
||||
def rabbit_user(name: str, password: str) -> dict:
|
||||
salt = secrets.token_bytes(4)
|
||||
digest = hashlib.sha256(salt + password.encode("utf-8")).digest()
|
||||
return {
|
||||
"name": name,
|
||||
"password_hash": base64.b64encode(salt + digest).decode("ascii"),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256",
|
||||
"tags": "",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.chmod(SECRET_DIR, 0o700)
|
||||
db_parts = urlsplit(read("database_url"))
|
||||
query = dict(parse_qsl(db_parts.query, keep_blank_values=True))
|
||||
query.update({"sslmode": "verify-full", "sslrootcert": "/run/secrets/ca_crt"})
|
||||
write(
|
||||
"database_url",
|
||||
urlunsplit((db_parts.scheme, db_parts.netloc, db_parts.path, urlencode(query), "")),
|
||||
replace=True,
|
||||
)
|
||||
worker_password_path = SECRET_DIR / "worker_db_password"
|
||||
if not worker_password_path.exists():
|
||||
write("worker_db_password", secrets.token_urlsafe(32))
|
||||
worker_password = read("worker_db_password")
|
||||
write(
|
||||
"worker_database_url",
|
||||
urlunsplit(
|
||||
(
|
||||
db_parts.scheme,
|
||||
f"messenger_worker:{worker_password}@{db_parts.hostname}:{db_parts.port}",
|
||||
db_parts.path,
|
||||
urlencode(query),
|
||||
"",
|
||||
)
|
||||
),
|
||||
replace=(SECRET_DIR / "worker_database_url").exists(),
|
||||
)
|
||||
|
||||
service_s3_credentials = {}
|
||||
for service in ("worker", "publisher", "migrate", "reconciler"):
|
||||
access_key_file = f"{service}_s3_access_key_id"
|
||||
secret_key_file = f"{service}_s3_secret_access_key"
|
||||
if not (SECRET_DIR / access_key_file).exists():
|
||||
write(access_key_file, service[:3] + secrets.token_hex(8))
|
||||
if not (SECRET_DIR / secret_key_file).exists():
|
||||
write(secret_key_file, secrets.token_urlsafe(40))
|
||||
service_s3_credentials[service] = (
|
||||
read(access_key_file),
|
||||
read(secret_key_file),
|
||||
)
|
||||
|
||||
credentials = {}
|
||||
users = []
|
||||
permissions = []
|
||||
for service, configure, write_access, read_access in (
|
||||
("worker", r"^$", r"^$", r"^attachments\.process\.v1$"),
|
||||
("publisher", r"^$", r"^amq\.default$", r"^$"),
|
||||
):
|
||||
password_file = f"rabbitmq_{service}_password"
|
||||
password_path = SECRET_DIR / password_file
|
||||
password = password_path.read_text(encoding="utf-8").strip() if password_path.exists() else secrets.token_urlsafe(32)
|
||||
if not password_path.exists():
|
||||
write(password_file, password)
|
||||
credentials[service] = password
|
||||
users.append(rabbit_user(service, password))
|
||||
permissions.append({
|
||||
"user": service,
|
||||
"vhost": "/",
|
||||
"configure": configure,
|
||||
"write": write_access,
|
||||
"read": read_access,
|
||||
})
|
||||
write(
|
||||
f"broker_{service}_url",
|
||||
f"amqps://{service}:{password}@rabbitmq:5671/%2f",
|
||||
replace=(SECRET_DIR / f"broker_{service}_url").exists(),
|
||||
)
|
||||
|
||||
definitions = {
|
||||
"users": users,
|
||||
"vhosts": [{"name": "/"}],
|
||||
"permissions": permissions,
|
||||
"queues": [
|
||||
{
|
||||
"name": "attachments.process.dead",
|
||||
"vhost": "/",
|
||||
"durable": True,
|
||||
"auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
|
||||
},
|
||||
{
|
||||
"name": "attachments.process.v1",
|
||||
"vhost": "/",
|
||||
"durable": True,
|
||||
"auto_delete": False,
|
||||
"arguments": {
|
||||
"x-queue-type": "quorum",
|
||||
"x-quorum-initial-group-size": 1,
|
||||
"x-dead-letter-exchange": "",
|
||||
"x-dead-letter-routing-key": "attachments.process.dead",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
definitions_path = SECRET_DIR / "rabbitmq_definitions.json"
|
||||
if definitions_path.exists():
|
||||
definitions_path.unlink()
|
||||
write("rabbitmq_definitions.json", json.dumps(definitions, separators=(",", ":")))
|
||||
os.chmod(definitions_path, 0o644)
|
||||
|
||||
config = (
|
||||
"default_user = messenger\n"
|
||||
f"default_pass = {read('rabbitmq_password')}\n"
|
||||
"listeners.tcp = none\n"
|
||||
"listeners.ssl.default = 5671\n"
|
||||
"ssl_options.cacertfile = /run/secrets/ca_crt\n"
|
||||
"ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n"
|
||||
"ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n"
|
||||
"ssl_options.verify = verify_peer\n"
|
||||
"ssl_options.fail_if_no_peer_cert = true\n"
|
||||
"ssl_options.versions.1 = tlsv1.2\n"
|
||||
"definitions.import_backend = local_filesystem\n"
|
||||
"definitions.local.path = /run/secrets/rabbitmq_definitions\n"
|
||||
)
|
||||
write("rabbitmq.conf", config, replace=True)
|
||||
identities = [
|
||||
{
|
||||
"name": "bootstrap-admin",
|
||||
"credentials": [{
|
||||
"accessKey": read("s3_admin_access_key_id"),
|
||||
"secretKey": read("s3_admin_secret_access_key"),
|
||||
}],
|
||||
"actions": ["Admin"],
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [{
|
||||
"accessKey": read("s3_access_key_id"),
|
||||
"secretKey": read("s3_secret_access_key"),
|
||||
}],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"List:messenger-attachments",
|
||||
"Read:messenger-variants",
|
||||
"Write:messenger-staging",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "messenger-worker",
|
||||
"credentials": [{
|
||||
"accessKey": service_s3_credentials["worker"][0],
|
||||
"secretKey": service_s3_credentials["worker"][1],
|
||||
}],
|
||||
"actions": [
|
||||
"Read:messenger-staging",
|
||||
"Write:messenger-staging",
|
||||
"Write:messenger-attachments",
|
||||
"Write:messenger-variants",
|
||||
],
|
||||
},
|
||||
*[
|
||||
{
|
||||
"name": f"messenger-{service}",
|
||||
"credentials": [{
|
||||
"accessKey": service_s3_credentials[service][0],
|
||||
"secretKey": service_s3_credentials[service][1],
|
||||
}],
|
||||
"actions": [],
|
||||
}
|
||||
for service in ("publisher", "migrate")
|
||||
],
|
||||
{
|
||||
"name": "messenger-reconciler",
|
||||
"credentials": [{
|
||||
"accessKey": service_s3_credentials["reconciler"][0],
|
||||
"secretKey": service_s3_credentials["reconciler"][1],
|
||||
}],
|
||||
"actions": [
|
||||
f"{action}:{bucket}"
|
||||
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants")
|
||||
for action in ("Read", "Write", "List")
|
||||
],
|
||||
},
|
||||
]
|
||||
write("s3.json", json.dumps({"identities": identities}, separators=(",", ":")), replace=True)
|
||||
iam_config_path = SECRET_DIR / "s3_iam.json"
|
||||
if not iam_config_path.exists():
|
||||
iam_config = {
|
||||
"sts": {
|
||||
"signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
|
||||
},
|
||||
"defaultEffect": "Deny",
|
||||
}
|
||||
write("s3_iam.json", json.dumps(iam_config, separators=(",", ":")))
|
||||
print("TLS endpoints and service-scoped broker credentials prepared locally; values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
interval=${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
grace=${RECONCILE_GRACE_SECONDS:-3600}
|
||||
|
||||
case "$interval" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$grace" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "$interval" -lt 60 ]; then
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$grace" -lt 60 ]; then
|
||||
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
while :
|
||||
do
|
||||
python3 src/reconcile.py --grace-seconds "$grace" --apply
|
||||
sleep "$interval"
|
||||
done
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
docker compose --env-file .env restart postgres rabbitmq s3-core s3
|
||||
|
||||
for service in postgres rabbitmq s3-core s3; do
|
||||
ready=0
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 90 ]; do
|
||||
ready=1
|
||||
containers=$(docker compose --env-file .env ps -q "$service")
|
||||
[ -n "$containers" ] || ready=0
|
||||
for container in $containers; do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}running{{end}}' "$container")
|
||||
[ "$status" = healthy ] || ready=0
|
||||
done
|
||||
[ "$ready" -eq 1 ] && break
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
[ "$ready" -eq 1 ] || {
|
||||
printf 'Service %s did not become healthy after restart.\n' "$service" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
docker compose --env-file .env restart api publisher worker reconciler
|
||||
printf '%s\n' 'Restarted API and application stores without changing volumes.'
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
matches=0
|
||||
target_container=
|
||||
for container in $(docker compose --env-file .env ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
matches=$((matches + 1))
|
||||
target_container=$container
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$matches" -ne 1 ]; then
|
||||
printf '%s\n' 'Could not resolve TARGET_INSTANCE_ID to exactly one API container.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker stop --time 2 "$target_container" >/dev/null
|
||||
@@ -1,19 +1,31 @@
|
||||
# Модель угроз — заполнить
|
||||
# Модель угроз — lab7
|
||||
|
||||
## Активы и акторы
|
||||
|
||||
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
|
||||
Активы: пароли пользователей, hashes сессий, сообщения и вложения, database/object-store/broker credentials, service identities, CA private key, OpenBao root token и unseal shares, telemetry. Акторы: внешний клиент, пользователь — участник чата, посторонний пользователь, скомпрометированный application service и локальный оператор.
|
||||
|
||||
Внешний атакующий может отправлять HTTPS запросы к loopback-published ingress. Пользователь действует только от своего Bearer session. Скомпрометированная служба получает только AppRole credential и секретный KV path, смонтированные этой службе. Оператор управляет PKI и OpenBao init/unseal. Модель не включает компрометацию Docker host, операционной системы или operator workstation.
|
||||
|
||||
## Границы доверия
|
||||
|
||||
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
|
||||
Внешняя граница — клиент → HAProxy по HTTPS. Внутренний API boundary — HAProxy → API по mTLS с проверкой CA chain и allowlisted URI SAN. OpenBao доступен по TLS и выдаёт статические secrets сервисным AppRole identities. Схема сетевых связей и протоколы приведены в [матрице доверия](../contracts/trust-matrix.md).
|
||||
|
||||
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO | TODO |
|
||||
|
||||
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
|
||||
| Клиент в сети пытается перехватить пароль при входе или Bearer token | Credentials передаются только аутентифицированному endpoint | HTTPS ingress, CA и hostname validation; открытый HTTP endpoint не опубликован | `make test-security` с корректным и пустым trust store | Внешний endpoint использует локальную учебную CA, которую клиент должен получить доверенным способом |
|
||||
| Клиент с сертификатом, подписанным доверенной CA, но с URI SAN другой службы, обращается к API | Только LB identity может установить соединение к API listener | mTLS + точное сравнение `spiffe://mtusi2026lab7/load-balancer` | `make test-internal-mtls` с identity `other-service` | Срок leaf ограничен 397 днями; автоматическая ротация не настроена |
|
||||
| Клиент без сертификата или с cert чужой CA пытается обойти ingress и обратиться к API listener | API не обслуживает неподтверждённое внутреннее соединение | `CERT_REQUIRED`, CA verification, API порт не опубликован на host | `make test-internal-mtls` без cert и с foreign CA; прямого host-порта нет | Проверка выполняется внутри контейнерной сети лабораторного Compose |
|
||||
| Пользователь вне чата запрашивает историю, metadata или content чужой attachment | Только член чата получает доступ к сообщениям и вложениям | Bearer session, ownership/membership checks, одинаковый 404 для чужих ресурсов | `make test` black-box сценарий ACL | Контракт не включает администрирование/редактирование состава участников |
|
||||
| При утечке PostgreSQL snapshot атакующий подбирает пароль | Пароль не хранится plaintext и имеет per-user salt | Argon2id `m=19456 KiB, t=2, p=1`; сессионные токены хранятся как SHA-256 hashes | `make test-primitives` | Параметры следует повторно измерить на целевом hardware |
|
||||
| Password spray пытается обойти лимит через несколько API-реплик и usernames | Серия неуспешных входов должна давать ограниченный ответ | PostgreSQL хранит fixed-window counters по SHA-256(username) и trusted source IP; sixth per-user attempt или 61st per-IP attempt в окне возвращает 429/Retry-After | `make test-login-limit` проверяет per-user threshold | Порог per-IP равен 60 попыткам за минуту |
|
||||
| Скомпрометированный API/worker запрашивает operator или чужие сервисные secrets | Application identity не получает root token/unseal shares и чужие KV paths | AppRole per service, KV v2 path-specific read policy, short TTL | `make test-openbao-acl` подтверждает собственный path и 403 для остальных сервисов | Поведение после истечения TTL не тестировалось; KV values статические |
|
||||
| Скомпрометированный worker пытается читать аккаунты/originals, перечислять buckets или публиковать свои jobs | Worker не получает DB users, S3 List/read originals/variants или broker publish privileges | Отдельная PostgreSQL-роль; S3 read/write на staging bucket, write на originals/variants; AMQP read только process queue | `make test-worker-isolation` проверяет grants, `AccessDenied` и broker `403`; `make test-worker-rotation` отзывает прежний SecretID и восстанавливает queued attachment | Проверен один worker и один queued recovery scenario на локальном Compose стенде |
|
||||
| Ошибка приложения/инструментации записывает token или тело сообщения в telemetry | Credentials и пользовательские данные не экспортируются как telemetry attributes | OTel instrumentation задаёт низкокардинальные route attributes; проверка кодом/конфигурацией | `make test` не является проверкой утечек telemetry; отдельный grep/inspection не приложен | Отдельный telemetry redaction test не запускался |
|
||||
|
||||
## Lifecycle и инциденты
|
||||
|
||||
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
|
||||
Оператор создаёт локальную CA и leaf сертификаты с `scripts/bootstrap-pki.py`; сертификаты и ключи лежат в ignored `secrets/pki/`. Root token/unseal shares создаются оператором при init и не передаются bootstrap скрипту через CLI arguments или app containers; `bootstrap-openbao.py` читает root token из локального файла, пишет policies и создаёт AppRole credentials под `secrets/openbao/`. Скрипт не печатает значения.
|
||||
|
||||
Ingress/API/OpenBao leaf certificates выпускаются на 397 дней; CA — на 3650 дней. Автоматический renewal/CA rollover не реализован. OpenBao AppRole SecretIDs имеют TTL 24 часа, сервисные tokens — 15 минут (max 1 час); приложение аутентифицируется и получает KV values при запуске. Ротация SecretID требует operator provisioning и перезапуска соответствующей службы. OpenBao state сохраняется в named volume; после рестарта оператор unseal-ит сервис по необходимости.
|
||||
|
||||
Runtime TLS проверен для внутренних сервисных каналов и telemetry backends; AppRole access isolation, worker ACL и восстановление после SecretID rotation зафиксированы отдельными сценариями. Автоматический rollover leaf-сертификатов не включён; оператор управляет их сроком и заменой.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API application package."""
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import ssl
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from .config import APP_HOST, APP_PORT
|
||||
from .handlers import MessengerHandler
|
||||
from .security import allowed_client_identity
|
||||
from .store import Store
|
||||
from .telemetry import initialize, start_runtime_metrics
|
||||
|
||||
LOGGER = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class MutualTLSServer(ThreadingHTTPServer):
|
||||
def __init__(self, address, handler):
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
context.load_cert_chain(os.environ["TLS_CERT_FILE"], os.environ["TLS_KEY_FILE"])
|
||||
context.load_verify_locations(cafile=os.environ["TLS_CA_FILE"])
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
self.tls_context = context
|
||||
client_cert_pem = open(os.environ["TLS_CLIENT_CERT_FILE"], encoding="ascii").read()
|
||||
client_cert_der = ssl.PEM_cert_to_DER_cert(client_cert_pem)
|
||||
self.client_certificate_fingerprint = hashlib.sha256(client_cert_der).digest()
|
||||
super().__init__(address, handler)
|
||||
|
||||
def get_request(self):
|
||||
raw_socket, address = super().get_request()
|
||||
try:
|
||||
connection = self.tls_context.wrap_socket(raw_socket, server_side=True)
|
||||
peer_certificate = connection.getpeercert(binary_form=True)
|
||||
fingerprint = hashlib.sha256(peer_certificate or b"").digest()
|
||||
if (
|
||||
not peer_certificate
|
||||
or not hmac.compare_digest(fingerprint, self.client_certificate_fingerprint)
|
||||
or not allowed_client_identity(connection.getpeercert())
|
||||
):
|
||||
connection.close()
|
||||
raise ssl.SSLError("client certificate identity is not authorized")
|
||||
return connection, address
|
||||
except (OSError, ssl.SSLError):
|
||||
raw_socket.close()
|
||||
raise
|
||||
|
||||
|
||||
def run_server():
|
||||
store = Store()
|
||||
store.open()
|
||||
start_runtime_metrics(store)
|
||||
try:
|
||||
handler = partial(MessengerHandler, store=store)
|
||||
server = MutualTLSServer((APP_HOST, APP_PORT), handler)
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
LOGGER.info("received signal %s, shutting down gracefully", signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def main():
|
||||
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-api"))
|
||||
run_server()
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3.bootstrap")
|
||||
|
||||
|
||||
def migrate_legacy_namespaces(client, source_bucket: str, targets: dict[str, str]) -> int:
|
||||
copied = 0
|
||||
for prefix, target_bucket in targets.items():
|
||||
pages = client.get_paginator("list_objects_v2").paginate(
|
||||
Bucket=source_bucket,
|
||||
Prefix=prefix,
|
||||
)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
key = item["Key"]
|
||||
try:
|
||||
target = client.head_object(Bucket=target_bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 404:
|
||||
raise
|
||||
source = client.get_object(Bucket=source_bucket, Key=key)
|
||||
body = source["Body"]
|
||||
try:
|
||||
content = body.read()
|
||||
finally:
|
||||
body.close()
|
||||
if len(content) != item["Size"]:
|
||||
raise RuntimeError(f"Legacy object read size mismatch for {key}")
|
||||
client.put_object(
|
||||
Bucket=target_bucket,
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentLength=len(content),
|
||||
ContentType=source.get("ContentType", "application/octet-stream"),
|
||||
Metadata=source.get("Metadata", {}),
|
||||
)
|
||||
target = client.head_object(Bucket=target_bucket, Key=key)
|
||||
copied += 1
|
||||
if (
|
||||
target["ContentLength"] != item["Size"]
|
||||
or target.get("ETag") != item.get("ETag")
|
||||
):
|
||||
raise RuntimeError(f"Legacy object copy verification failed for {key}")
|
||||
return copied
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
direct = os.getenv(name)
|
||||
if direct:
|
||||
return direct
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
def main():
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
|
||||
verify=os.getenv("S3_CA_FILE", True),
|
||||
config=Config(
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 3},
|
||||
client_cert=(os.getenv("S3_CLIENT_CERT_FILE"), os.getenv("S3_CLIENT_KEY_FILE")),
|
||||
),
|
||||
)
|
||||
buckets = (
|
||||
os.getenv("S3_BUCKET", "messenger-attachments"),
|
||||
os.getenv("S3_STAGING_BUCKET", "messenger-staging"),
|
||||
os.getenv("S3_VARIANTS_BUCKET", "messenger-variants"),
|
||||
)
|
||||
for bucket in buckets:
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
except ClientError as create_error:
|
||||
if create_error.response.get("Error", {}).get("Code") != "BucketAlreadyOwnedByYou":
|
||||
raise
|
||||
client.put_bucket_acl(Bucket=bucket, ACL="private")
|
||||
copied = migrate_legacy_namespaces(
|
||||
client,
|
||||
buckets[0],
|
||||
{
|
||||
"staging/": buckets[1],
|
||||
"variants/": buckets[2],
|
||||
},
|
||||
)
|
||||
LOGGER.info("private S3 buckets bootstrapped; legacy objects copied=%d", copied)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
main()
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import socket
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
|
||||
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
|
||||
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
|
||||
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
|
||||
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
|
||||
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
|
||||
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_STAGING_BUCKET = os.getenv("S3_STAGING_BUCKET", "messenger-staging")
|
||||
S3_VARIANTS_BUCKET = os.getenv("S3_VARIANTS_BUCKET", "messenger-variants")
|
||||
|
||||
|
||||
_BAO_LOCK = threading.Lock()
|
||||
_BAO_CACHE = {}
|
||||
|
||||
|
||||
def _read_secret_file(path_variable: str) -> str:
|
||||
file_path = os.getenv(path_variable)
|
||||
if not file_path:
|
||||
raise RuntimeError(f"Set {path_variable}")
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def _bao_secrets() -> dict:
|
||||
with _BAO_LOCK:
|
||||
if "values" in _BAO_CACHE:
|
||||
return _BAO_CACHE["values"]
|
||||
address = os.getenv("BAO_ADDR")
|
||||
ca_file = os.getenv("BAO_CACERT")
|
||||
role_id = _read_secret_file("BAO_ROLE_ID_FILE")
|
||||
secret_id = _read_secret_file("BAO_SECRET_ID_FILE")
|
||||
secret_path = os.getenv("BAO_SECRET_PATH")
|
||||
if not address or not ca_file or not secret_path:
|
||||
raise RuntimeError("OpenBao service authentication requires BAO_ADDR, BAO_CACERT and BAO_SECRET_PATH")
|
||||
context = ssl.create_default_context(cafile=ca_file)
|
||||
|
||||
def request(path: str, payload: dict, token: str = "", method: str = "") -> dict:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
data = json.dumps(payload).encode("utf-8") if payload else None
|
||||
req = urllib.request.Request(
|
||||
address.rstrip("/") + "/v1/" + path,
|
||||
data=data,
|
||||
headers=headers,
|
||||
method=method or None,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=context, timeout=5) as response:
|
||||
return json.loads(response.read(1024 * 1024))
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
||||
raise RuntimeError(f"OpenBao request failed ({type(exc).__name__})") from None
|
||||
|
||||
auth = request("auth/approle/login", {"role_id": role_id, "secret_id": secret_id})
|
||||
token = auth.get("auth", {}).get("client_token")
|
||||
if not token:
|
||||
raise RuntimeError("OpenBao AppRole login returned no service token")
|
||||
response = request(f"secret/data/{secret_path.lstrip('/')}", {}, token, "GET")
|
||||
values = response.get("data", {}).get("data")
|
||||
if not isinstance(values, dict):
|
||||
raise RuntimeError("OpenBao returned an invalid secret payload")
|
||||
_BAO_CACHE["values"] = values
|
||||
return values
|
||||
|
||||
|
||||
def secret_value(name: str, default: str = "") -> str:
|
||||
if os.getenv("BAO_ROLE_ID_FILE") or os.getenv("BAO_ROLE_ID"):
|
||||
values = _bao_secrets()
|
||||
if name in values and isinstance(values[name], str):
|
||||
return values[name]
|
||||
raise RuntimeError(f"OpenBao path does not contain {name}")
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
if default:
|
||||
return default
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
|
||||
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
if SESSION_TTL_SECONDS < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
|
||||
if IDEMPOTENCY_TTL_SECONDS < 86400:
|
||||
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
|
||||
raise RuntimeError("Database timeouts must be positive")
|
||||
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
|
||||
raise RuntimeError("Database pool settings must be positive")
|
||||
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
|
||||
raise RuntimeError("HTTP limits must be positive")
|
||||
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
|
||||
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
|
||||
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
|
||||
if MAX_CONCURRENT_UPLOADS < 1:
|
||||
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
|
||||
if UPLOAD_TIMEOUT_SECONDS < 1:
|
||||
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
|
||||
@@ -0,0 +1,26 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class CacheUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyConflictError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AttachmentNotReadyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ObjectMissingError(Exception):
|
||||
pass
|
||||
+653
@@ -0,0 +1,653 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import tempfile
|
||||
import uuid
|
||||
import warnings
|
||||
import zlib
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import unicodedata
|
||||
|
||||
import psycopg
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .config import (
|
||||
IDEMPOTENCY_KEY_RE,
|
||||
INSTANCE_ID,
|
||||
MAX_CONCURRENT_UPLOADS,
|
||||
MAX_IMAGE_PIXELS,
|
||||
MAX_INFLIGHT_REQUESTS,
|
||||
MAX_REQUEST_BODY_BYTES,
|
||||
MAX_UPLOAD_BYTES,
|
||||
UPLOAD_TIMEOUT_SECONDS,
|
||||
USER_RE,
|
||||
)
|
||||
from .errors import (
|
||||
ApiError,
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
ObjectMissingError,
|
||||
)
|
||||
from .security import PasswordHashCapacityError, hash_password, isoformat, valid_uuid
|
||||
from .store import Store
|
||||
from .telemetry import REQUEST_ID, record_http_request, tracer
|
||||
|
||||
LOGGER = logging.getLogger("messenger.http")
|
||||
|
||||
|
||||
class RequestGuard:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
|
||||
def acquire(self):
|
||||
with self.lock:
|
||||
if self.inflight >= MAX_INFLIGHT_REQUESTS:
|
||||
raise ApiError(503, "unavailable", "Server is busy")
|
||||
self.inflight += 1
|
||||
|
||||
def release(self):
|
||||
with self.lock:
|
||||
self.inflight = max(0, self.inflight - 1)
|
||||
|
||||
|
||||
REQUEST_GUARD = RequestGuard()
|
||||
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
|
||||
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
def __init__(self, request, client_address, server, store: Store):
|
||||
self.store = store
|
||||
super().__init__(request, client_address, server)
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(15)
|
||||
|
||||
def log_message(self, format: str, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
path = urlsplit(self.path).path
|
||||
route = self._route_template(path)
|
||||
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
parent_context = propagate.extract({key.lower(): value for key, value in self.headers.items()})
|
||||
started = time.perf_counter()
|
||||
token = REQUEST_ID.set(request_id)
|
||||
self._response_status = 500
|
||||
if route.startswith("/health/"):
|
||||
try:
|
||||
self._dispatch_request(method, request_id)
|
||||
finally:
|
||||
record_http_request(method, route, self._response_status, time.perf_counter() - started)
|
||||
REQUEST_ID.reset(token)
|
||||
return
|
||||
with tracer().start_as_current_span(
|
||||
f"{method} {route}",
|
||||
context=parent_context,
|
||||
kind=SpanKind.SERVER,
|
||||
attributes={
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"http.request_id": request_id,
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
self._dispatch_request(method, request_id)
|
||||
finally:
|
||||
duration = time.perf_counter() - started
|
||||
span.set_attribute("http.response.status_code", self._response_status)
|
||||
if self._response_status >= 500:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
record_http_request(method, route, self._response_status, duration)
|
||||
REQUEST_ID.reset(token)
|
||||
|
||||
@staticmethod
|
||||
def _route_template(path: str) -> str:
|
||||
if path in ("/health/live", "/health/ready", "/api/v1/users", "/api/v1/auth/sessions",
|
||||
"/api/v1/auth/sessions/current", "/api/v1/users/me", "/api/v1/chats"):
|
||||
return path
|
||||
if path.startswith("/api/v1/chats/") and path.endswith("/attachments"):
|
||||
return "/api/v1/chats/{chat_id}/attachments"
|
||||
if path.startswith("/api/v1/chats/") and path.endswith("/messages"):
|
||||
return "/api/v1/chats/{chat_id}/messages"
|
||||
if path.startswith("/api/v1/chats/"):
|
||||
return "/api/v1/chats/{chat_id}"
|
||||
if path.startswith("/api/v1/attachments/") and path.endswith("/content"):
|
||||
return "/api/v1/attachments/{attachment_id}/content"
|
||||
if path.startswith("/api/v1/attachments/"):
|
||||
return "/api/v1/attachments/{attachment_id}"
|
||||
return "unmatched"
|
||||
|
||||
def _dispatch_request(self, method: str, request_id: str):
|
||||
acquired = False
|
||||
try:
|
||||
REQUEST_GUARD.acquire()
|
||||
acquired = True
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if path == "/health/live" and method == "GET":
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/health/ready" and method == "GET":
|
||||
try:
|
||||
with suppress_instrumentation():
|
||||
self.store.ready()
|
||||
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
|
||||
raise ApiError(503, "unavailable", "A required storage service is unavailable")
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/api/v1/users" and method == "POST":
|
||||
self._handle_register(request_id)
|
||||
elif path == "/api/v1/auth/sessions" and method == "POST":
|
||||
self._handle_create_session(request_id)
|
||||
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
self._handle_revoke_session(request_id)
|
||||
elif path == "/api/v1/users/me" and method == "GET":
|
||||
self._send_json(200, self._get_actor(request_id), request_id)
|
||||
elif path == "/api/v1/chats" and method == "POST":
|
||||
self._handle_create_chat(request_id)
|
||||
elif path == "/api/v1/chats" and method == "GET":
|
||||
self._handle_list_chats(query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
self._handle_upload_attachment(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/attachments/") and method == "GET":
|
||||
attachment_path = path.removeprefix("/api/v1/attachments/")
|
||||
if attachment_path.endswith("/content"):
|
||||
attachment_id = attachment_path.removesuffix("/content")
|
||||
self._handle_download_attachment(attachment_id, query, request_id)
|
||||
else:
|
||||
self._handle_get_attachment(attachment_path, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
if method == "POST":
|
||||
self._handle_create_message(chat_id, request_id)
|
||||
else:
|
||||
self._handle_list_messages(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and method == "GET":
|
||||
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
|
||||
else:
|
||||
raise ApiError(404, "not_found", "Object not found")
|
||||
except ApiError as exc:
|
||||
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
|
||||
except CacheUnavailableError:
|
||||
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
|
||||
except PasswordHashCapacityError:
|
||||
self._send_error(
|
||||
503,
|
||||
"unavailable",
|
||||
"Password verification capacity is busy",
|
||||
request_id,
|
||||
{"Retry-After": "1"},
|
||||
)
|
||||
except IdempotencyConflictError:
|
||||
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
|
||||
except AttachmentNotReadyError:
|
||||
self._send_error(409, "conflict", "Attachment is not ready", request_id)
|
||||
except ObjectMissingError:
|
||||
LOGGER.error("attachment object missing for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
|
||||
except (BotoCoreError, ClientError):
|
||||
LOGGER.warning("object storage unavailable for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
self._send_error(503, "unavailable", "Database is unavailable", request_id)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
self._send_error(409, "conflict", "Username already exists", request_id)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
self._send_error(400, "invalid_request", "Unknown member", request_id)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
self._set_response_status(499)
|
||||
self.close_connection = True
|
||||
LOGGER.info("client disconnected before response for request %s", request_id)
|
||||
except Exception as exc:
|
||||
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
|
||||
self._send_error(500, "internal_error", "Internal server error", request_id)
|
||||
finally:
|
||||
if acquired:
|
||||
REQUEST_GUARD.release()
|
||||
|
||||
@staticmethod
|
||||
def _trace_context():
|
||||
carrier = {}
|
||||
propagate.inject(carrier)
|
||||
return {"traceparent": carrier.get("traceparent"), "tracestate": carrier.get("tracestate")}
|
||||
|
||||
def _set_response_status(self, status: int) -> None:
|
||||
self._response_status = status
|
||||
|
||||
def _read_json(self, request_id: str) -> Any:
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Missing Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size < 0:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size > MAX_REQUEST_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Request body too large")
|
||||
raw = self.rfile.read(size)
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Empty body")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON")
|
||||
|
||||
def _get_actor(self, request_id: str) -> Dict[str, Any]:
|
||||
header = self.headers.get("Authorization", "")
|
||||
scheme, separator, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not separator or not token or " " in token:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
user = self.store.session_user(token)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
return user
|
||||
|
||||
def _handle_register(self, request_id: str):
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
|
||||
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
|
||||
username = body["username"]
|
||||
display_name = body["display_name"]
|
||||
password = body["password"]
|
||||
if not isinstance(username, str) or not USER_RE.fullmatch(username):
|
||||
raise ApiError(400, "invalid_request", "Invalid username")
|
||||
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid display_name")
|
||||
if not isinstance(password, str) or not (12 <= len(password) <= 128):
|
||||
raise ApiError(400, "invalid_request", "Invalid password")
|
||||
user = self.store.create_user(username, display_name, hash_password(password))
|
||||
self._send_json(201, user, request_id)
|
||||
|
||||
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
|
||||
value = self.headers.get("Authorization", "")
|
||||
scheme, separator, encoded = value.partition(" ")
|
||||
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
if scheme.lower() != "basic" or not separator or not encoded:
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
if not USER_RE.fullmatch(username) or not 12 <= len(password) <= 128:
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
return username, password
|
||||
|
||||
def _handle_create_session(self, request_id: str):
|
||||
username, password = self._basic_credentials(request_id)
|
||||
source_ip = self.headers.get("X-Real-IP", self.client_address[0])
|
||||
retry_after = self.store.consume_login_attempt(username, source_ip)
|
||||
if retry_after is not None:
|
||||
raise ApiError(
|
||||
429,
|
||||
"rate_limited",
|
||||
"Too many login attempts",
|
||||
{"Retry-After": str(retry_after)},
|
||||
)
|
||||
user = self.store.authenticate(username, password)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
self.store.clear_login_attempts(username)
|
||||
token, expires_at = self.store.create_session(user["id"])
|
||||
self._send_json(
|
||||
201,
|
||||
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
|
||||
request_id,
|
||||
)
|
||||
|
||||
def _handle_revoke_session(self, request_id: str):
|
||||
self._get_actor(request_id)
|
||||
token = self.headers["Authorization"].partition(" ")[2]
|
||||
if not self.store.revoke_session(token):
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
self._set_response_status(204)
|
||||
self.send_response(204)
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.end_headers()
|
||||
|
||||
def _handle_create_chat(self, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
|
||||
raise ApiError(400, "invalid_request", "Expected title and member_ids")
|
||||
title, member_ids = body["title"], body["member_ids"]
|
||||
if not isinstance(title, str) or not title.strip() or len(title) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid title")
|
||||
if (
|
||||
not isinstance(member_ids, list)
|
||||
or not 1 <= len(member_ids) <= 100
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
|
||||
or len(member_ids) != len(set(member_ids))
|
||||
or actor["id"] not in member_ids
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
||||
chat = self.store.create_chat(title, member_ids, actor["id"])
|
||||
self._send_json(201, chat, request_id)
|
||||
|
||||
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _handle_get_chat(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat = self.store.get_chat(chat_id, actor["id"])
|
||||
if chat is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(200, chat, request_id)
|
||||
|
||||
def _handle_create_message(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
idempotency_key = self.headers.get("Idempotency-Key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
|
||||
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
|
||||
text = body["text"]
|
||||
attachment_ids = body.get("attachment_ids", [])
|
||||
if (
|
||||
not isinstance(text, str)
|
||||
or len(text) > 4000
|
||||
or not isinstance(attachment_ids, list)
|
||||
or len(attachment_ids) > 10
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
|
||||
or len(attachment_ids) != len(set(attachment_ids))
|
||||
or (not text.strip() and not attachment_ids)
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid message text")
|
||||
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
|
||||
canonical_body = json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
message = self.store.create_idempotent_message(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
text,
|
||||
attachment_ids,
|
||||
idempotency_key,
|
||||
canonical_body,
|
||||
)
|
||||
if message is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(201, message, request_id)
|
||||
|
||||
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
|
||||
if declared_type not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None or self.headers.get("Transfer-Encoding"):
|
||||
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size <= 0:
|
||||
raise ApiError(415, "unsupported_media_type", "Image is empty")
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
self.close_connection = True
|
||||
remaining = min(size, MAX_UPLOAD_BYTES + 1)
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
while remaining:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
break
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
try:
|
||||
chunk = self.rfile.read(min(64 * 1024, remaining))
|
||||
except (TimeoutError, OSError):
|
||||
LOGGER.info("stopped bounded drain of oversized upload")
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
|
||||
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
|
||||
digest = hashlib.sha256()
|
||||
bytes_read = 0
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
try:
|
||||
while bytes_read < size:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
raise ApiError(408, "invalid_request", "Image upload timed out")
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
|
||||
if not chunk:
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body")
|
||||
content.write(chunk)
|
||||
digest.update(chunk)
|
||||
bytes_read += len(chunk)
|
||||
except (TimeoutError, OSError):
|
||||
content.close()
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body") from None
|
||||
|
||||
try:
|
||||
content.seek(0)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
|
||||
image.verify()
|
||||
content.seek(0)
|
||||
with IMAGE_DECODE_GUARD:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
image.load()
|
||||
content.seek(0)
|
||||
return content, actual_type, size, digest.hexdigest()
|
||||
except ApiError:
|
||||
content.close()
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
content.close()
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
|
||||
content.close()
|
||||
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
|
||||
|
||||
@staticmethod
|
||||
def _valid_filename(filename: str) -> bool:
|
||||
return (
|
||||
1 <= len(filename) <= 255
|
||||
and not any(char in ("/", "\\", "\x00") for char in filename)
|
||||
and not any(unicodedata.category(char) == "Cc" for char in filename)
|
||||
)
|
||||
|
||||
def _handle_upload_attachment(
|
||||
self, chat_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
if not self.store.can_access_chat(chat_id, actor["id"]):
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
filenames = query.get("filename", [])
|
||||
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
|
||||
raise ApiError(400, "invalid_request", "Invalid filename")
|
||||
if not UPLOAD_GUARD.acquire(blocking=False):
|
||||
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
|
||||
try:
|
||||
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
|
||||
try:
|
||||
attachment = self.store.create_attachment(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
filenames[0],
|
||||
content_type,
|
||||
content,
|
||||
size,
|
||||
digest,
|
||||
**self._trace_context(),
|
||||
)
|
||||
finally:
|
||||
content.close()
|
||||
finally:
|
||||
UPLOAD_GUARD.release()
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(202, attachment, request_id)
|
||||
|
||||
def _handle_get_attachment(self, attachment_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
self._send_json(200, attachment, request_id)
|
||||
|
||||
def _handle_download_attachment(
|
||||
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
variants = query.get("variant", ["original"])
|
||||
if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment variant")
|
||||
variant = variants[0]
|
||||
if variant != "original" and variant not in ("thumbnail", "crop", "optimized"):
|
||||
raise ApiError(404, "not_found", "Attachment variant not found")
|
||||
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant)
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
content_type, content = result
|
||||
self._set_response_status(200)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Cache-Control", "private, no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
items, next_cursor = result
|
||||
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _send_json(
|
||||
self,
|
||||
status: int,
|
||||
payload: Dict[str, Any],
|
||||
request_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._set_response_status(status)
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_error(
|
||||
self,
|
||||
status: int,
|
||||
code: str,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
|
||||
import pika
|
||||
from .config import secret_value
|
||||
|
||||
QUEUE_NAME = "attachments.process.v1"
|
||||
DEAD_QUEUE_NAME = "attachments.process.dead"
|
||||
LEASE_NETWORK_BUDGET_SECONDS = 150
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
return secret_value("BROKER_URL")
|
||||
|
||||
|
||||
def connect() -> pika.BlockingConnection:
|
||||
parameters = pika.URLParameters(broker_url())
|
||||
context = ssl.create_default_context(cafile=os.environ["BROKER_CA_FILE"])
|
||||
context.load_cert_chain(
|
||||
os.environ["BROKER_CLIENT_CERT_FILE"],
|
||||
os.environ["BROKER_CLIENT_KEY_FILE"],
|
||||
)
|
||||
parameters.ssl_options = pika.SSLOptions(context, parameters.host)
|
||||
parameters.connection_attempts = 3
|
||||
parameters.retry_delay = 1
|
||||
parameters.socket_timeout = 5
|
||||
parameters.blocked_connection_timeout = 10
|
||||
return pika.BlockingConnection(parameters)
|
||||
|
||||
|
||||
def encode_job(job: dict) -> bytes:
|
||||
return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level="INFO")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
store.migrate()
|
||||
worker_password = Path(os.environ["WORKER_DB_PASSWORD_FILE"]).read_text(encoding="utf-8").strip()
|
||||
store.configure_worker_role(worker_password)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import (
|
||||
S3_ACCESS_KEY_ID,
|
||||
S3_BUCKET,
|
||||
S3_ENDPOINT,
|
||||
S3_REGION,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
S3_STAGING_BUCKET,
|
||||
S3_VARIANTS_BUCKET,
|
||||
)
|
||||
from .errors import ObjectMissingError
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3")
|
||||
|
||||
|
||||
def bucket_for_key(key: str) -> str:
|
||||
if key.startswith("staging/"):
|
||||
return S3_STAGING_BUCKET
|
||||
if key.startswith("original/"):
|
||||
return S3_BUCKET
|
||||
if key.startswith("variants/"):
|
||||
return S3_VARIANTS_BUCKET
|
||||
raise ValueError("Object key is outside configured storage namespaces")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
verify=os.getenv("S3_CA_FILE", True),
|
||||
config=Config(
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
client_cert=(
|
||||
os.getenv("S3_CLIENT_CERT_FILE"),
|
||||
os.getenv("S3_CLIENT_KEY_FILE"),
|
||||
),
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
def put(
|
||||
self,
|
||||
key: str,
|
||||
content: Union[bytes, BinaryIO],
|
||||
content_type: str,
|
||||
size_bytes: Optional[int] = None,
|
||||
) -> None:
|
||||
if size_bytes is None:
|
||||
size_bytes = len(content)
|
||||
self.client.put_object(
|
||||
Bucket=bucket_for_key(key),
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentLength=size_bytes,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key: str, max_bytes: int) -> bytes:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=bucket_for_key(key), Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404", "NotFound"):
|
||||
raise ObjectMissingError from exc
|
||||
raise
|
||||
body = response["Body"]
|
||||
try:
|
||||
value = body.read(max_bytes + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if len(value) > max_bytes:
|
||||
raise RuntimeError("Stored object exceeds configured safety limit")
|
||||
return value
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=bucket_for_key(key), Key=key)
|
||||
|
||||
def ready(self) -> bool:
|
||||
self.client.head_bucket(Bucket=S3_BUCKET)
|
||||
return True
|
||||
|
||||
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
||||
for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET):
|
||||
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"]
|
||||
@@ -0,0 +1,4 @@
|
||||
PIPELINE_QUALITY = {
|
||||
1: {"jpeg": 85, "webp": 80},
|
||||
2: {"jpeg": 80, "webp": 75},
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, encode_job
|
||||
from .security import isoformat
|
||||
from .store import Store
|
||||
from .telemetry import initialize, tracer
|
||||
|
||||
LOGGER = logging.getLogger("messenger.publisher")
|
||||
|
||||
|
||||
def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
|
||||
with store.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
with suppress_instrumentation():
|
||||
row = store.claim_outbox(conn)
|
||||
if row is None:
|
||||
return False
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(row["job_id"]),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": row["pipeline_version"],
|
||||
"attempt": row["attempt"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
parent_context = propagate.extract(
|
||||
{
|
||||
key: row.get(key)
|
||||
for key in ("traceparent", "tracestate")
|
||||
if row.get(key) is not None
|
||||
}
|
||||
)
|
||||
with tracer().start_as_current_span(
|
||||
"attachment.publish",
|
||||
context=parent_context,
|
||||
kind=SpanKind.PRODUCER,
|
||||
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "publish"},
|
||||
):
|
||||
carrier = {}
|
||||
propagate.inject(carrier)
|
||||
payload.update(carrier)
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=QUEUE_NAME,
|
||||
body=encode_job(payload),
|
||||
properties=pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(row["id"]),
|
||||
type="attachment.process.v1",
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-publisher"))
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
while True:
|
||||
connection = None
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
store.recover_expired_attachment_jobs(
|
||||
int(os.getenv("JOB_MAX_ATTEMPTS", "5")),
|
||||
int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS,
|
||||
)
|
||||
if not publish_pending(store, channel):
|
||||
time.sleep(1)
|
||||
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
|
||||
LOGGER.exception("outbox publishing cycle failed")
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
connection.close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.reconcile")
|
||||
|
||||
|
||||
def reconcile(
|
||||
store: Store,
|
||||
grace_seconds: int,
|
||||
apply_changes: bool,
|
||||
restore_dir: Optional[Path] = None,
|
||||
):
|
||||
started = time.monotonic()
|
||||
metadata = store.reconciliation_snapshot()
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
found = set()
|
||||
orphans = []
|
||||
for key, last_modified in store.objects.objects():
|
||||
found.add(key)
|
||||
if key not in metadata and last_modified <= cutoff:
|
||||
orphans.append(key)
|
||||
|
||||
missing = [
|
||||
row for key, row in metadata.items()
|
||||
if key not in found and row.get("reconcile_type", "original") == "original"
|
||||
]
|
||||
restore_candidates = []
|
||||
if restore_dir is not None:
|
||||
root = restore_dir.resolve(strict=True)
|
||||
for row in missing:
|
||||
source = restore_dir / str(row["id"])
|
||||
try:
|
||||
resolved = source.resolve(strict=True)
|
||||
if (
|
||||
resolved.parent != root
|
||||
or not resolved.is_file()
|
||||
or resolved.stat().st_size != row["size_bytes"]
|
||||
):
|
||||
continue
|
||||
content = resolved.read_bytes()
|
||||
except OSError:
|
||||
continue
|
||||
if (
|
||||
len(content) == row["size_bytes"]
|
||||
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
|
||||
):
|
||||
restore_candidates.append((row, content))
|
||||
|
||||
restored = 0
|
||||
if apply_changes:
|
||||
for key in orphans:
|
||||
store.objects.delete(key)
|
||||
restorable_ids = set()
|
||||
for row, content in restore_candidates:
|
||||
store.objects.put(row["object_key"], content, row["content_type"])
|
||||
store.clear_missing_object(row["id"])
|
||||
restorable_ids.add(row["id"])
|
||||
restored += 1
|
||||
for row in missing:
|
||||
if row["id"] not in restorable_ids:
|
||||
store.record_missing_object(row["id"])
|
||||
for key, row in metadata.items():
|
||||
if key in found and row.get("reconcile_type", "original") == "original":
|
||||
store.clear_missing_object(row["id"])
|
||||
|
||||
unresolved_missing = len(missing) - restored
|
||||
return {
|
||||
"mode": "apply" if apply_changes else "dry-run",
|
||||
"grace_seconds": grace_seconds,
|
||||
"objects_seen": len(found),
|
||||
"metadata_seen": len({str(row["id"]) for row in metadata.values()}),
|
||||
"orphan_objects": len([key for key in found if key not in metadata]),
|
||||
"stale_orphan_candidates": len(orphans),
|
||||
"deleted_orphans": len(orphans) if apply_changes else 0,
|
||||
"missing_objects": len(missing),
|
||||
"unresolved_missing_objects": unresolved_missing,
|
||||
"restorable_missing_objects": len(restore_candidates),
|
||||
"restored_missing_objects": restored,
|
||||
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
|
||||
parser.add_argument(
|
||||
"--restore-dir",
|
||||
type=Path,
|
||||
help="optional trusted backup directory with exact original bytes named by attachment UUID",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error("--grace-seconds must be at least 60")
|
||||
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
if result["unresolved_missing_objects"]:
|
||||
LOGGER.error(
|
||||
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
|
||||
result["unresolved_missing_objects"],
|
||||
)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Explicitly replay a failed attachment or start a newer pipeline version."
|
||||
)
|
||||
parser.add_argument("--attachment-id", required=True, type=uuid.UUID)
|
||||
parser.add_argument(
|
||||
"--pipeline-version",
|
||||
required=True,
|
||||
type=int,
|
||||
help="pipeline version implemented by the deployed worker",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.pipeline_version < 1:
|
||||
parser.error("--pipeline-version must be positive")
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
from argon2.low_level import Type
|
||||
|
||||
|
||||
_ARGON2_MEMORY_KIB = int(os.getenv("ARGON2_MEMORY_KIB", "19456"))
|
||||
_ARGON2_ITERATIONS = int(os.getenv("ARGON2_ITERATIONS", "2"))
|
||||
_ARGON2_PARALLELISM = int(os.getenv("ARGON2_PARALLELISM", "1"))
|
||||
_PASSWORD_HASH_CONCURRENCY = int(os.getenv("PASSWORD_HASH_CONCURRENCY", "4"))
|
||||
_PASSWORD_HASH_WAIT_SECONDS = float(os.getenv("PASSWORD_HASH_WAIT_SECONDS", "0.1"))
|
||||
if (
|
||||
_ARGON2_MEMORY_KIB < 19456
|
||||
or _ARGON2_MEMORY_KIB > 65536
|
||||
or not 2 <= _ARGON2_ITERATIONS <= 4
|
||||
or not 1 <= _ARGON2_PARALLELISM <= 4
|
||||
):
|
||||
raise RuntimeError("Argon2id settings must be 19456..65536 KiB, t=2..4, and p=1..4")
|
||||
if not 1 <= _PASSWORD_HASH_CONCURRENCY <= 4 or _PASSWORD_HASH_WAIT_SECONDS <= 0:
|
||||
raise RuntimeError("Password hash concurrency must be 1..4 with a positive wait timeout")
|
||||
|
||||
_PASSWORD_HASHER = PasswordHasher(
|
||||
time_cost=_ARGON2_ITERATIONS,
|
||||
memory_cost=_ARGON2_MEMORY_KIB,
|
||||
parallelism=_ARGON2_PARALLELISM,
|
||||
type=Type.ID,
|
||||
)
|
||||
_PASSWORD_HASH_SLOTS = threading.BoundedSemaphore(_PASSWORD_HASH_CONCURRENCY)
|
||||
|
||||
|
||||
class PasswordHashCapacityError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _password_hash_slot():
|
||||
if not _PASSWORD_HASH_SLOTS.acquire(timeout=_PASSWORD_HASH_WAIT_SECONDS):
|
||||
raise PasswordHashCapacityError("password hashing capacity is temporarily exhausted")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_PASSWORD_HASH_SLOTS.release()
|
||||
|
||||
|
||||
def isoformat(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def valid_uuid(value: str) -> bool:
|
||||
try:
|
||||
uuid.UUID(value)
|
||||
return True
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
with _password_hash_slot():
|
||||
return _PASSWORD_HASHER.hash(password)
|
||||
|
||||
|
||||
def _verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
if stored.startswith("$argon2id$"):
|
||||
parts = stored.split("$")
|
||||
if len(parts) != 6 or parts[2] != "v=19":
|
||||
return False
|
||||
parameters = dict(part.split("=", 1) for part in parts[3].split(","))
|
||||
memory, iterations, parallelism = (
|
||||
int(parameters["m"]),
|
||||
int(parameters["t"]),
|
||||
int(parameters["p"]),
|
||||
)
|
||||
if not (19456 <= memory <= 65536 and 2 <= iterations <= 4 and 1 <= parallelism <= 4):
|
||||
return False
|
||||
return _PASSWORD_HASHER.verify(stored, password)
|
||||
# Existing lab6 accounts can log in once and be upgraded to Argon2id.
|
||||
scheme, n, r, p, salt_text, hash_text = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(expected),
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (
|
||||
InvalidHashError,
|
||||
VerificationError,
|
||||
VerifyMismatchError,
|
||||
ValueError,
|
||||
TypeError,
|
||||
UnicodeError,
|
||||
IndexError,
|
||||
KeyError,
|
||||
):
|
||||
return False
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
with _password_hash_slot():
|
||||
return _verify_password(password, stored)
|
||||
|
||||
|
||||
def password_hash_needs_upgrade(stored: str) -> bool:
|
||||
if not stored.startswith("$argon2id$"):
|
||||
return True
|
||||
try:
|
||||
return _PASSWORD_HASHER.check_needs_rehash(stored)
|
||||
except (InvalidHashError, VerificationError):
|
||||
return True
|
||||
|
||||
|
||||
def allowed_client_identity(cert: dict) -> bool:
|
||||
return ("URI", "spiffe://mtusi2026lab7/load-balancer") in cert.get("subjectAltName", ())
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.app import main
|
||||
else:
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+934
@@ -0,0 +1,934 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
|
||||
import redis
|
||||
from psycopg import sql
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .config import (
|
||||
DB_LOCK_TIMEOUT_MS,
|
||||
DB_POOL_MAX_SIZE,
|
||||
DB_POOL_TIMEOUT_SECONDS,
|
||||
DB_STATEMENT_TIMEOUT_MS,
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
secret_value,
|
||||
)
|
||||
from .errors import (
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
ObjectMissingError,
|
||||
)
|
||||
from .object_store import ObjectStore
|
||||
from .pipeline_versions import PIPELINE_QUALITY
|
||||
from .security import (
|
||||
hash_password,
|
||||
isoformat,
|
||||
password_hash_needs_upgrade,
|
||||
utc_now,
|
||||
valid_uuid,
|
||||
verify_password,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger("messenger.store")
|
||||
_DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
return secret_value("DATABASE_URL")
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
|
||||
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
conninfo=database_url(),
|
||||
min_size=1,
|
||||
max_size=DB_POOL_MAX_SIZE,
|
||||
timeout=DB_POOL_TIMEOUT_SECONDS,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": (
|
||||
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
|
||||
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
|
||||
"-c idle_in_transaction_session_timeout=10000"
|
||||
),
|
||||
},
|
||||
open=False,
|
||||
)
|
||||
cache_url = os.getenv("CACHE_URL")
|
||||
self.cache = (
|
||||
redis.Redis.from_url(
|
||||
cache_url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=0.25,
|
||||
socket_timeout=0.25,
|
||||
retry_on_timeout=False,
|
||||
max_connections=16,
|
||||
)
|
||||
if cache_url
|
||||
else None
|
||||
)
|
||||
self.objects = ObjectStore()
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
if self.cache is not None:
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self) -> bool:
|
||||
with self.pool.connection(timeout=2) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
self.objects.ready()
|
||||
return True
|
||||
|
||||
def migrate(self):
|
||||
migrations = Path(__file__).resolve().parents[1] / "migrations"
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for migration in sorted(migrations.glob("*.sql")):
|
||||
if migration.name in applied:
|
||||
continue
|
||||
conn.execute(migration.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
|
||||
|
||||
def configure_worker_role(self, password: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker'"
|
||||
).fetchone()
|
||||
if exists:
|
||||
statement = sql.SQL("ALTER ROLE messenger_worker LOGIN PASSWORD {}").format(
|
||||
sql.Literal(password)
|
||||
)
|
||||
else:
|
||||
statement = sql.SQL("CREATE ROLE messenger_worker LOGIN PASSWORD {}").format(
|
||||
sql.Literal(password)
|
||||
)
|
||||
conn.execute(statement)
|
||||
conn.execute("GRANT CONNECT ON DATABASE messenger TO messenger_worker")
|
||||
conn.execute("GRANT USAGE ON SCHEMA public TO messenger_worker")
|
||||
conn.execute(
|
||||
"GRANT SELECT (id, job_id, pipeline_version, attempt, status, lease_until, "
|
||||
"staging_key, object_key, content_type, size_bytes, sha256) ON attachments "
|
||||
"TO messenger_worker"
|
||||
)
|
||||
conn.execute(
|
||||
"GRANT UPDATE (status, lease_until, updated_at, variants, error_code, attempt) "
|
||||
"ON attachments TO messenger_worker"
|
||||
)
|
||||
conn.execute(
|
||||
"GRANT INSERT (id, job_id, attachment_id, pipeline_version, attempt, available_at, "
|
||||
"traceparent, tracestate) ON attachment_outbox TO messenger_worker"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def user(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"attachment_ids": attachment_ids or [],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
variants = row.get("variants") or []
|
||||
public_variants = [
|
||||
{
|
||||
name: variant[name]
|
||||
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
|
||||
}
|
||||
for variant in variants
|
||||
]
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"].strip(),
|
||||
"status": row.get("status", "queued"),
|
||||
"variants": public_variants,
|
||||
"error_code": row.get("error_code"),
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"updated_at": isoformat(row["updated_at"]),
|
||||
}
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return self.user(row)
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
return self.user(row) if row else None
|
||||
|
||||
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash "
|
||||
"FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
stored_hash = row["password_hash"] if row else _DUMMY_PASSWORD_HASH
|
||||
if not verify_password(password, stored_hash) or not row:
|
||||
return None
|
||||
if password_hash_needs_upgrade(stored_hash):
|
||||
upgraded_hash = hash_password(password)
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s",
|
||||
(upgraded_hash, row["id"], stored_hash),
|
||||
)
|
||||
return self.user(row)
|
||||
|
||||
def consume_login_attempt(self, username: str, source_ip: str) -> Optional[int]:
|
||||
user_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
|
||||
ip_key = hashlib.sha256(b"ip\0" + source_ip.encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
def bump(key: bytes) -> Dict[str, Any]:
|
||||
return conn.execute(
|
||||
"""
|
||||
INSERT INTO login_rate_limits (username_key, window_started_at, attempts)
|
||||
VALUES (%s, clock_timestamp(), 1)
|
||||
ON CONFLICT (username_key) DO UPDATE SET
|
||||
attempts = CASE
|
||||
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
|
||||
THEN 1 ELSE login_rate_limits.attempts + 1
|
||||
END,
|
||||
window_started_at = CASE
|
||||
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
|
||||
THEN clock_timestamp() ELSE login_rate_limits.window_started_at
|
||||
END
|
||||
RETURNING attempts,
|
||||
GREATEST(1, ceil(extract(epoch FROM
|
||||
(window_started_at + interval '1 minute' - clock_timestamp()))))::integer AS retry_after
|
||||
""",
|
||||
(key,),
|
||||
).fetchone()
|
||||
|
||||
user_row = bump(user_key)
|
||||
ip_row = bump(ip_key)
|
||||
conn.execute(
|
||||
"DELETE FROM login_rate_limits WHERE window_started_at < clock_timestamp() - interval '1 hour'"
|
||||
)
|
||||
retries = []
|
||||
if user_row["attempts"] > 5:
|
||||
retries.append(user_row["retry_after"])
|
||||
if ip_row["attempts"] > 60:
|
||||
retries.append(ip_row["retry_after"])
|
||||
return max(retries) if retries else None
|
||||
|
||||
def clear_login_attempts(self, username: str) -> None:
|
||||
username_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("DELETE FROM login_rate_limits WHERE username_key = %s", (username_key,))
|
||||
|
||||
def create_session(self, user_id: str) -> Tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
token_hash = hashlib.sha256(token.encode("ascii")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash, user_id, expires_at),
|
||||
)
|
||||
return token, expires_at
|
||||
|
||||
@staticmethod
|
||||
def _session_lock_key(token_hash: bytes) -> int:
|
||||
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
|
||||
|
||||
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
cache_key = "messenger:session:" + token_hash.hex()
|
||||
if self.cache is not None:
|
||||
try:
|
||||
cached = self.cache.get(cache_key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
row = conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
|
||||
"FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
user = self.user(row)
|
||||
if self.cache is not None:
|
||||
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
|
||||
if ttl_ms > 0:
|
||||
try:
|
||||
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
|
||||
except redis.RedisError:
|
||||
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
|
||||
return user
|
||||
|
||||
def revoke_session(self, token: str) -> bool:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
if self.cache is not None:
|
||||
try:
|
||||
self.cache.delete("messenger:session:" + token_hash.hex())
|
||||
except redis.RedisError as exc:
|
||||
raise CacheUnavailableError from exc
|
||||
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
|
||||
return result.rowcount == 1
|
||||
|
||||
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
|
||||
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
|
||||
raise ValueError("invalid members")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
|
||||
(title,),
|
||||
).fetchone()
|
||||
for member_id in member_ids:
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
(row["id"], member_id),
|
||||
)
|
||||
return self.chat(row, member_ids)
|
||||
|
||||
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members m ON m.chat_id = c.id "
|
||||
"WHERE c.id = %s AND m.user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return self.chat(row, [str(item["user_id"]) for item in members])
|
||||
|
||||
def list_chats(
|
||||
self, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
||||
where = "own.user_id = %s"
|
||||
params: List[Any] = [user_id]
|
||||
if cursor is not None:
|
||||
try:
|
||||
raw = base64.b64decode(
|
||||
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
|
||||
altchars=b"-_",
|
||||
validate=True,
|
||||
)
|
||||
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
|
||||
if cursor_user != user_id or not valid_uuid(chat_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None or created_at.utcoffset() is None:
|
||||
raise ValueError("cursor timestamp must include a timezone")
|
||||
where += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params.extend((created_at, chat_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members own ON own.chat_id = c.id "
|
||||
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
result = []
|
||||
for row in page:
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return result, next_cursor
|
||||
|
||||
def create_idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
attachment_ids: List[str],
|
||||
idempotency_key: str,
|
||||
canonical_body: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
|
||||
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
|
||||
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
return None
|
||||
if attachment_ids:
|
||||
attachments = conn.execute(
|
||||
"SELECT id, status FROM attachments "
|
||||
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
|
||||
(chat_id, attachment_ids),
|
||||
).fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
return None
|
||||
if any(item["status"] == "failed" for item in attachments):
|
||||
raise AttachmentNotReadyError
|
||||
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
|
||||
existing = conn.execute(
|
||||
"SELECT request_hash, response, expires_at "
|
||||
"FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
|
||||
"FOR UPDATE",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if existing and existing["expires_at"] > utc_now():
|
||||
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
|
||||
raise IdempotencyConflictError
|
||||
return existing["response"]
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
)
|
||||
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (chat_id, sender_id, text) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
|
||||
(chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
for ordinal, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
|
||||
"VALUES (%s, %s, %s)",
|
||||
(row["id"], attachment_id, ordinal),
|
||||
)
|
||||
message = self.message(row, attachment_ids)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency_records "
|
||||
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
|
||||
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
|
||||
(
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency_key,
|
||||
request_hash,
|
||||
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
|
||||
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
|
||||
),
|
||||
)
|
||||
return message
|
||||
|
||||
def list_messages(
|
||||
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
|
||||
with self.pool.connection() as conn:
|
||||
is_member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not is_member:
|
||||
return None
|
||||
where = "chat_id = %s"
|
||||
params: List[Any] = [chat_id]
|
||||
if cursor:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
|
||||
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
|
||||
if decoded_chat != chat_id or not valid_uuid(message_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
where += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend((created_at, message_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
ids_by_message = {}
|
||||
if rows:
|
||||
links = conn.execute(
|
||||
"SELECT message_id, attachment_id FROM message_attachments "
|
||||
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
|
||||
([row["id"] for row in rows],),
|
||||
).fetchall()
|
||||
for link in links:
|
||||
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
|
||||
|
||||
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone() is not None
|
||||
|
||||
def create_attachment(
|
||||
self,
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
content: BinaryIO,
|
||||
size_bytes: int,
|
||||
digest: str,
|
||||
traceparent: Optional[str] = None,
|
||||
tracestate: Optional[str] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not self.can_access_chat(chat_id, owner_id):
|
||||
return None
|
||||
attachment_id = uuid.uuid4()
|
||||
job_id = uuid.uuid4()
|
||||
staging_key = f"staging/{attachment_id}"
|
||||
object_key = f"original/{attachment_id}"
|
||||
content.seek(0)
|
||||
self.objects.put(staging_key, content, content_type, size_bytes)
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"object_key, staging_key, job_id) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, variants, error_code, created_at, updated_at",
|
||||
(
|
||||
attachment_id,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
digest,
|
||||
object_key,
|
||||
staging_key,
|
||||
job_id,
|
||||
),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
|
||||
"VALUES (%s, %s, %s, 1, 1, %s, %s)",
|
||||
(uuid.uuid4(), job_id, attachment_id, traceparent, tracestate),
|
||||
)
|
||||
return self.attachment(row)
|
||||
except Exception:
|
||||
try:
|
||||
self.objects.delete(staging_key)
|
||||
except Exception as cleanup_error:
|
||||
LOGGER.error(
|
||||
"attachment staging compensation failed for %s (%s)",
|
||||
attachment_id,
|
||||
type(cleanup_error).__name__,
|
||||
)
|
||||
raise
|
||||
|
||||
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
|
||||
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
return self.attachment(row) if row else None
|
||||
|
||||
def download_attachment(
|
||||
self, attachment_id: str, user_id: str, variant: str = "original"
|
||||
) -> Optional[Tuple[str, bytes]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
if row["status"] != "ready":
|
||||
raise AttachmentNotReadyError
|
||||
if variant == "original":
|
||||
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
|
||||
else:
|
||||
match = next(
|
||||
(item for item in (row["variants"] or []) if item["name"] == variant),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
return None
|
||||
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
|
||||
return content_type, self.objects.get(key, size_bytes)
|
||||
|
||||
def claim_attachment_job(
|
||||
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
|
||||
"object_key, content_type, size_bytes, sha256 "
|
||||
"FROM attachments WHERE id = %s FOR UPDATE",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
not row
|
||||
or str(row["job_id"]) != job_id
|
||||
or row["pipeline_version"] != pipeline_version
|
||||
or row["status"] in ("ready", "failed")
|
||||
or row["attempt"] != attempt
|
||||
):
|
||||
return None
|
||||
if row["status"] == "processing" and row["lease_until"] is not None:
|
||||
if row["lease_until"] > utc_now():
|
||||
return None
|
||||
return conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', "
|
||||
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
|
||||
"updated_at = clock_timestamp() "
|
||||
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
|
||||
"size_bytes, sha256, attempt, pipeline_version",
|
||||
(lease_seconds, attachment_id),
|
||||
).fetchone()
|
||||
|
||||
def complete_attachment_job(
|
||||
self,
|
||||
attachment_id: str,
|
||||
job_id: str,
|
||||
pipeline_version: int,
|
||||
attempt: int,
|
||||
variants: List[Dict[str, Any]],
|
||||
) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
updated = conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
|
||||
"lease_until = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
|
||||
"AND status = 'processing' AND lease_until > clock_timestamp()",
|
||||
(
|
||||
json.dumps(variants, separators=(",", ":")),
|
||||
attachment_id,
|
||||
job_id,
|
||||
pipeline_version,
|
||||
attempt,
|
||||
),
|
||||
)
|
||||
return updated.rowcount == 1
|
||||
|
||||
def fail_attachment_job(
|
||||
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
|
||||
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
|
||||
traceparent: Optional[str] = None, tracestate: Optional[str] = None,
|
||||
) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT status, attempt FROM attachments "
|
||||
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
|
||||
(attachment_id, job_id, pipeline_version),
|
||||
).fetchone()
|
||||
if not row or row["status"] != "processing" or row["attempt"] != attempt:
|
||||
return
|
||||
if retry_delay_seconds is not None and attempt < max_attempts:
|
||||
next_attempt = attempt + 1
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
|
||||
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(next_attempt, error_code, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
|
||||
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) "
|
||||
"ON CONFLICT (job_id, attempt) DO NOTHING",
|
||||
(
|
||||
uuid.uuid4(), job_id, attachment_id, pipeline_version,
|
||||
next_attempt, retry_delay_seconds, traceparent, tracestate,
|
||||
),
|
||||
)
|
||||
else:
|
||||
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
|
||||
"updated_at = clock_timestamp() WHERE id = %s",
|
||||
(final_code, attachment_id),
|
||||
)
|
||||
|
||||
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
|
||||
if pipeline_version not in PIPELINE_QUALITY:
|
||||
raise ValueError("pipeline version is not implemented by this deployment")
|
||||
with self.pool.connection() as conn:
|
||||
current = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
|
||||
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if current is None:
|
||||
raise ValueError("attachment not found")
|
||||
if pipeline_version < current["pipeline_version"]:
|
||||
raise ValueError("pipeline version cannot move backwards")
|
||||
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
|
||||
raise ValueError("same-version replay is allowed only for failed attachments")
|
||||
if current["status"] not in ("queued", "processing", "ready", "failed"):
|
||||
raise ValueError("attachment cannot be replayed in its current state")
|
||||
|
||||
try:
|
||||
content = self.objects.get(current["object_key"], current["size_bytes"])
|
||||
except ObjectMissingError as object_error:
|
||||
try:
|
||||
content = self.objects.get(current["staging_key"], current["size_bytes"])
|
||||
except ObjectMissingError:
|
||||
raise object_error
|
||||
if (
|
||||
len(content) != current["size_bytes"]
|
||||
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
|
||||
):
|
||||
raise ValueError("stored original does not match attachment metadata")
|
||||
|
||||
prior_job_id = str(current["job_id"])
|
||||
next_job_id = uuid.uuid4()
|
||||
staging_key = f"staging/{attachment_id}/{next_job_id}"
|
||||
self.objects.put(staging_key, content, current["content_type"], len(content))
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
locked = conn.execute(
|
||||
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
|
||||
"WHERE id = %s FOR UPDATE",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
locked is None
|
||||
or str(locked["job_id"]) != prior_job_id
|
||||
or locked["pipeline_version"] != current["pipeline_version"]
|
||||
or locked["attempt"] != current["attempt"]
|
||||
or locked["status"] != current["status"]
|
||||
):
|
||||
raise RuntimeError("attachment changed while preparing replay")
|
||||
conn.execute(
|
||||
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
|
||||
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
|
||||
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s",
|
||||
(next_job_id, pipeline_version, staging_key, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt) "
|
||||
"VALUES (%s, %s, %s, %s, 1)",
|
||||
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_pipeline_events "
|
||||
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
|
||||
"VALUES (%s, %s, %s, %s, %s, 'replay')",
|
||||
(
|
||||
attachment_id,
|
||||
prior_job_id,
|
||||
next_job_id,
|
||||
current["pipeline_version"],
|
||||
pipeline_version,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
self.objects.delete(staging_key)
|
||||
except Exception as cleanup_error:
|
||||
LOGGER.error(
|
||||
"replay staging cleanup failed for %s (%s)",
|
||||
attachment_id,
|
||||
type(cleanup_error).__name__,
|
||||
)
|
||||
raise
|
||||
return {
|
||||
"attachment_id": str(attachment_id),
|
||||
"previous_job_id": prior_job_id,
|
||||
"job_id": str(next_job_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": 1,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
|
||||
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
|
||||
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
if row["attempt"] >= max_attempts:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
|
||||
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
continue
|
||||
next_attempt = row["attempt"] + 1
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
|
||||
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
|
||||
(next_attempt, row["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox "
|
||||
"(id, job_id, attachment_id, pipeline_version, attempt) "
|
||||
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
|
||||
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
def claim_outbox(self, conn):
|
||||
return conn.execute(
|
||||
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
|
||||
"o.traceparent, o.tracestate, "
|
||||
"a.created_at FROM attachment_outbox o "
|
||||
"JOIN attachments a ON a.id = o.attachment_id "
|
||||
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
|
||||
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
|
||||
).fetchone()
|
||||
|
||||
def queue_snapshot(self) -> Tuple[int, float]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS depth, "
|
||||
"COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age "
|
||||
"FROM attachments WHERE status = 'queued'"
|
||||
).fetchone()
|
||||
return int(row["depth"]), float(row["oldest_age"])
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
|
||||
"size_bytes, sha256, content_type FROM attachments"
|
||||
).fetchall()
|
||||
expected = {}
|
||||
for row in rows:
|
||||
if row["status"] == "ready":
|
||||
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
|
||||
for variant in row["variants"] or []:
|
||||
expected[variant["object_key"]] = {
|
||||
**row,
|
||||
"object_key": variant["object_key"],
|
||||
"size_bytes": variant["size_bytes"],
|
||||
"sha256": variant["sha256"],
|
||||
"content_type": variant["content_type"],
|
||||
"reconcile_type": "variant",
|
||||
}
|
||||
elif row["status"] in ("queued", "processing"):
|
||||
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
|
||||
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
|
||||
for name, extension in (
|
||||
("thumbnail", "jpg"),
|
||||
("crop", "jpg"),
|
||||
("optimized", "webp"),
|
||||
):
|
||||
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
|
||||
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
|
||||
return expected
|
||||
|
||||
def record_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
|
||||
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
|
||||
"DO UPDATE SET last_seen_at = clock_timestamp()",
|
||||
(attachment_id,),
|
||||
)
|
||||
|
||||
def clear_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM reconciliation_incidents "
|
||||
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
|
||||
(attachment_id,),
|
||||
)
|
||||
@@ -0,0 +1,300 @@
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import psycopg
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON, ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
||||
|
||||
REQUEST_ID = contextvars.ContextVar("request_id", default="")
|
||||
_lock = threading.Lock()
|
||||
_runtime = {
|
||||
"pool_used": 0,
|
||||
"pool_waiting": 0,
|
||||
"queue_depth": 0,
|
||||
"queue_oldest_seconds": 0.0,
|
||||
"worker_busy": 0,
|
||||
"worker_capacity": 0,
|
||||
"cpu": 0.0,
|
||||
"memory": 0,
|
||||
}
|
||||
_provider = None
|
||||
_meter = None
|
||||
_service_name = "messenger"
|
||||
_enabled = True
|
||||
_request_counter = None
|
||||
_request_duration = None
|
||||
_job_counter = None
|
||||
_job_duration = None
|
||||
_time_to_ready = None
|
||||
_queue_depth = None
|
||||
_queue_oldest = None
|
||||
_pool_used = None
|
||||
_pool_waiting = None
|
||||
_worker_busy = None
|
||||
_worker_capacity = None
|
||||
_process_cpu = None
|
||||
_process_memory = None
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
converter = time.gmtime
|
||||
|
||||
def format(self, record):
|
||||
span_context = trace.get_current_span().get_span_context()
|
||||
document = {
|
||||
"timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
|
||||
"severity": record.levelname,
|
||||
"service.name": _service_name,
|
||||
"event": record.getMessage(),
|
||||
"request_id": REQUEST_ID.get(),
|
||||
}
|
||||
if span_context.is_valid:
|
||||
document["trace_id"] = format(span_context.trace_id, "032x")
|
||||
document["span_id"] = format(span_context.span_id, "016x")
|
||||
if record.exc_info:
|
||||
document["exception.type"] = record.exc_info[0].__name__
|
||||
return json.dumps(document, ensure_ascii=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _sampler():
|
||||
if os.getenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") == "always_off":
|
||||
return ALWAYS_OFF
|
||||
if os.getenv("OTEL_TRACES_SAMPLER", "") == "always_on":
|
||||
return ALWAYS_ON
|
||||
ratio = float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
||||
if not 0 <= ratio <= 1:
|
||||
raise ValueError("OTEL_TRACES_SAMPLER_ARG must be between 0 and 1")
|
||||
return ParentBased(TraceIdRatioBased(ratio))
|
||||
|
||||
|
||||
def initialize(service_name: str) -> None:
|
||||
global _enabled
|
||||
global _service_name
|
||||
global _provider, _meter
|
||||
global _request_counter, _request_duration, _job_counter, _job_duration, _time_to_ready
|
||||
global _queue_depth, _queue_oldest, _pool_used, _pool_waiting
|
||||
global _worker_busy, _worker_capacity, _process_cpu, _process_memory
|
||||
|
||||
enabled = os.getenv("OTEL_SDK_DISABLED", "false").lower() != "true"
|
||||
_enabled = enabled
|
||||
_service_name = service_name
|
||||
instance_id = os.getenv("INSTANCE_ID", socket.gethostname())
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": service_name,
|
||||
"service.version": os.getenv("SERVICE_VERSION", "lab6"),
|
||||
"service.instance.id": instance_id,
|
||||
"deployment.environment.name": os.getenv("APP_ENV", "lab"),
|
||||
"host.name": socket.gethostname(),
|
||||
"os.type": platform.system().lower(),
|
||||
}
|
||||
)
|
||||
if enabled:
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
|
||||
_provider = TracerProvider(resource=resource, sampler=_sampler())
|
||||
_provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=endpoint.rstrip("/") + "/v1/traces"),
|
||||
max_queue_size=4096,
|
||||
max_export_batch_size=512,
|
||||
schedule_delay_millis=1000,
|
||||
export_timeout_millis=10000,
|
||||
)
|
||||
)
|
||||
trace.set_tracer_provider(_provider)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint.rstrip("/") + "/v1/metrics"),
|
||||
export_interval_millis=10000,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[reader]))
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint.rstrip("/") + "/v1/logs"),
|
||||
max_queue_size=4096,
|
||||
max_export_batch_size=512,
|
||||
schedule_delay_millis=1000,
|
||||
export_timeout_millis=10000,
|
||||
)
|
||||
)
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otel_handler.setFormatter(JsonLogFormatter())
|
||||
else:
|
||||
trace.set_tracer_provider(TracerProvider(resource=resource, sampler=ALWAYS_OFF))
|
||||
metrics.set_meter_provider(MeterProvider(resource=resource))
|
||||
otel_handler = None
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(os.getenv("LOG_LEVEL", "INFO"))
|
||||
formatter = JsonLogFormatter()
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(formatter)
|
||||
root.handlers[:] = [stream_handler]
|
||||
if otel_handler is not None:
|
||||
root.addHandler(otel_handler)
|
||||
logging.getLogger("pika").setLevel(logging.WARNING)
|
||||
|
||||
if enabled:
|
||||
try:
|
||||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||||
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
|
||||
|
||||
BotocoreInstrumentor().instrument()
|
||||
PsycopgInstrumentor().instrument()
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("OTel database and S3 instrumentation dependencies are required") from exc
|
||||
|
||||
_meter = metrics.get_meter("messenger.telemetry")
|
||||
_request_counter = _meter.create_counter("messenger.http.requests", unit="{request}")
|
||||
_request_duration = _meter.create_histogram("messenger.http.duration", unit="s")
|
||||
_job_counter = _meter.create_counter("messenger.jobs.processed", unit="{job}")
|
||||
_job_duration = _meter.create_histogram("messenger.jobs.duration", unit="s")
|
||||
_time_to_ready = _meter.create_histogram("messenger.attachments.time_to_ready", unit="s")
|
||||
|
||||
def observe(name, attributes=None):
|
||||
def callback(_options):
|
||||
with _lock:
|
||||
return [metrics.Observation(_runtime[name], attributes=attributes or {})]
|
||||
|
||||
return callback
|
||||
|
||||
_queue_depth = _meter.create_observable_gauge(
|
||||
"messenger.queue.depth",
|
||||
callbacks=[observe("queue_depth", {"queue.name": "attachments.process.v1"})],
|
||||
unit="{job}",
|
||||
)
|
||||
_queue_oldest = _meter.create_observable_gauge(
|
||||
"messenger.queue.oldest_age",
|
||||
callbacks=[observe("queue_oldest_seconds", {"queue.name": "attachments.process.v1"})],
|
||||
unit="s",
|
||||
)
|
||||
_pool_used = _meter.create_observable_gauge(
|
||||
"messenger.db.pool.connections.used",
|
||||
callbacks=[observe("pool_used", {"pool.name": "primary"})],
|
||||
unit="{connection}",
|
||||
)
|
||||
_pool_waiting = _meter.create_observable_gauge(
|
||||
"messenger.db.pool.requests.waiting",
|
||||
callbacks=[observe("pool_waiting", {"pool.name": "primary"})],
|
||||
unit="{request}",
|
||||
)
|
||||
_worker_busy = _meter.create_observable_gauge(
|
||||
"messenger.worker.busy", callbacks=[observe("worker_busy")], unit="{worker}"
|
||||
)
|
||||
_worker_capacity = _meter.create_observable_gauge(
|
||||
"messenger.worker.capacity", callbacks=[observe("worker_capacity")], unit="{worker}"
|
||||
)
|
||||
_process_cpu = _meter.create_observable_gauge(
|
||||
"messenger.process.cpu.utilization", callbacks=[observe("cpu")], unit="1"
|
||||
)
|
||||
_process_memory = _meter.create_observable_gauge(
|
||||
"messenger.process.memory.usage", callbacks=[observe("memory")], unit="By"
|
||||
)
|
||||
|
||||
|
||||
def tracer():
|
||||
return trace.get_tracer("messenger.application")
|
||||
|
||||
|
||||
def record_http_request(method: str, route: str, status: int, duration: float) -> None:
|
||||
attributes = {
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"status_class": f"{status // 100}xx",
|
||||
}
|
||||
if _request_counter is not None:
|
||||
_request_counter.add(1, attributes)
|
||||
_request_duration.record(duration, attributes)
|
||||
if _enabled and not route.startswith("/health/"):
|
||||
logging.getLogger("messenger.http").info(
|
||||
"http request completed method=%s route=%s status_class=%s",
|
||||
method,
|
||||
route,
|
||||
f"{status // 100}xx",
|
||||
)
|
||||
|
||||
|
||||
def record_job(outcome: str, duration: float, time_to_ready: Optional[float] = None) -> None:
|
||||
attributes = {"job.type": "attachment.process.v1", "outcome": outcome}
|
||||
if _job_counter is not None:
|
||||
_job_counter.add(1, attributes)
|
||||
_job_duration.record(duration, attributes)
|
||||
if time_to_ready is not None:
|
||||
_time_to_ready.record(time_to_ready, {"outcome": outcome})
|
||||
|
||||
|
||||
def update_runtime(**values) -> None:
|
||||
with _lock:
|
||||
_runtime.update(values)
|
||||
|
||||
|
||||
def start_runtime_metrics(
|
||||
store, worker_capacity: int = 0, include_queue_snapshot: bool = True
|
||||
) -> Optional[threading.Thread]:
|
||||
if not _enabled:
|
||||
return None
|
||||
|
||||
def refresh():
|
||||
try:
|
||||
import psutil
|
||||
|
||||
process = psutil.Process()
|
||||
process.cpu_percent(interval=None)
|
||||
while True:
|
||||
try:
|
||||
stats = store.pool.get_stats()
|
||||
if include_queue_snapshot:
|
||||
with suppress_instrumentation():
|
||||
queue_depth, oldest_age = store.queue_snapshot()
|
||||
cpu = process.cpu_percent(interval=1) / 100.0
|
||||
memory = process.memory_info().rss
|
||||
runtime = dict(
|
||||
pool_used=max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)),
|
||||
pool_waiting=stats.get("requests_waiting", 0),
|
||||
worker_capacity=worker_capacity,
|
||||
cpu=cpu,
|
||||
memory=memory,
|
||||
)
|
||||
if include_queue_snapshot:
|
||||
runtime.update(queue_depth=queue_depth, queue_oldest_seconds=oldest_age)
|
||||
update_runtime(**runtime)
|
||||
except (psycopg.Error, OSError, RuntimeError, ValueError):
|
||||
logging.getLogger("messenger.telemetry").exception("runtime metric sample failed")
|
||||
time.sleep(9)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
logging.getLogger("messenger.telemetry").exception("runtime metric sampler stopped")
|
||||
raise
|
||||
|
||||
thread = threading.Thread(target=refresh, name="runtime-metrics", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def worker_started() -> None:
|
||||
with _lock:
|
||||
_runtime["worker_busy"] += 1
|
||||
|
||||
|
||||
def worker_stopped() -> None:
|
||||
with _lock:
|
||||
_runtime["worker_busy"] = max(0, _runtime["worker_busy"] - 1)
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
import warnings
|
||||
import zlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable, Optional
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from opentelemetry import propagate
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from .config import MAX_IMAGE_PIXELS
|
||||
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect
|
||||
from .pipeline_versions import PIPELINE_QUALITY
|
||||
from .store import Store
|
||||
from .telemetry import (
|
||||
initialize,
|
||||
record_job,
|
||||
start_runtime_metrics,
|
||||
tracer,
|
||||
worker_started,
|
||||
worker_stopped,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger("messenger.worker")
|
||||
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
||||
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
|
||||
|
||||
|
||||
class InvalidAttachment(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedPipeline(InvalidAttachment):
|
||||
pass
|
||||
|
||||
|
||||
def _rgb_image(content: bytes) -> Image.Image:
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(io.BytesIO(content)) as source:
|
||||
if source.format not in ("PNG", "JPEG"):
|
||||
raise InvalidAttachment("unsupported_image")
|
||||
width, height = source.size
|
||||
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
|
||||
raise InvalidAttachment("image_too_large")
|
||||
source.load()
|
||||
image = ImageOps.exif_transpose(source)
|
||||
try:
|
||||
if "A" in image.getbands() or "transparency" in image.info:
|
||||
with image.convert("RGBA") as rgba:
|
||||
background = Image.new("RGB", rgba.size, "white")
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return image.convert("RGB")
|
||||
finally:
|
||||
if image is not source:
|
||||
image.close()
|
||||
except InvalidAttachment:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError,
|
||||
UnidentifiedImageError, ValueError, zlib.error) as exc:
|
||||
raise InvalidAttachment("invalid_image") from exc
|
||||
|
||||
|
||||
def _encode(image: Image.Image, image_format: str, **options) -> bytes:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format=image_format, **options)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1):
|
||||
quality = PIPELINE_QUALITY.get(pipeline_version)
|
||||
if quality is None:
|
||||
raise UnsupportedPipeline("unsupported_pipeline_version")
|
||||
image = _rgb_image(content)
|
||||
try:
|
||||
variants = []
|
||||
thumbnail = image.copy()
|
||||
try:
|
||||
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True)
|
||||
variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes))
|
||||
finally:
|
||||
thumbnail.close()
|
||||
|
||||
crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
|
||||
try:
|
||||
crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True)
|
||||
variants.append(("crop", "image/jpeg", 128, 128, crop_bytes))
|
||||
finally:
|
||||
crop.close()
|
||||
|
||||
optimized = image.copy()
|
||||
try:
|
||||
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6)
|
||||
variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes))
|
||||
finally:
|
||||
optimized.close()
|
||||
|
||||
result = []
|
||||
for name, content_type, width, height, data in variants:
|
||||
extension = "webp" if content_type == "image/webp" else "jpg"
|
||||
key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}"
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"object_key": key,
|
||||
"content": data,
|
||||
}
|
||||
)
|
||||
return result
|
||||
finally:
|
||||
image.close()
|
||||
|
||||
|
||||
class ProcessingTimeout(TimeoutError):
|
||||
pass
|
||||
|
||||
|
||||
def _variant_process(connection, attachment_id, content, pipeline_version, builder):
|
||||
try:
|
||||
connection.send(("ok", builder(attachment_id, content, pipeline_version)))
|
||||
except InvalidAttachment as exc:
|
||||
connection.send(("invalid", str(exc)))
|
||||
except Exception as exc:
|
||||
connection.send(("error", type(exc).__name__))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _build_variants_bounded(
|
||||
attachment_id: str,
|
||||
content: bytes,
|
||||
pipeline_version: int,
|
||||
timeout_seconds: int,
|
||||
builder=build_variants,
|
||||
):
|
||||
if timeout_seconds < 1:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(
|
||||
target=_variant_process,
|
||||
args=(sender, attachment_id, content, pipeline_version, builder),
|
||||
)
|
||||
process.start()
|
||||
sender.close()
|
||||
try:
|
||||
if not receiver.poll(timeout_seconds):
|
||||
process.terminate()
|
||||
process.join(timeout=2)
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join()
|
||||
raise ProcessingTimeout("image processing exceeded its time budget")
|
||||
try:
|
||||
result, value = receiver.recv()
|
||||
except EOFError as exc:
|
||||
raise RuntimeError("image processing process exited without a result") from exc
|
||||
process.join(timeout=2)
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
if result == "invalid":
|
||||
raise InvalidAttachment(value)
|
||||
if result == "error":
|
||||
raise RuntimeError(f"image processing failed in subprocess ({value})")
|
||||
return value
|
||||
finally:
|
||||
receiver.close()
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join()
|
||||
|
||||
|
||||
def _job(payload: bytes):
|
||||
value = json.loads(payload)
|
||||
required = {
|
||||
"schema_version", "job_id", "type", "attachment_id",
|
||||
"pipeline_version", "attempt", "created_at",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}):
|
||||
raise ValueError("invalid job fields")
|
||||
if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1":
|
||||
raise ValueError("invalid job schema")
|
||||
uuid.UUID(value["job_id"])
|
||||
uuid.UUID(value["attachment_id"])
|
||||
if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1:
|
||||
raise ValueError("invalid pipeline version")
|
||||
if not isinstance(value["attempt"], int) or value["attempt"] < 1:
|
||||
raise ValueError("invalid job attempt")
|
||||
return value
|
||||
|
||||
|
||||
def _process(
|
||||
store: Store,
|
||||
job: dict,
|
||||
fault_injector: Optional[Callable[[str], None]] = None,
|
||||
) -> str:
|
||||
def inject(point: str) -> None:
|
||||
if fault_injector is not None:
|
||||
fault_injector(point)
|
||||
|
||||
attachment_id = job["attachment_id"]
|
||||
row = store.claim_attachment_job(
|
||||
attachment_id,
|
||||
job["job_id"],
|
||||
job["pipeline_version"],
|
||||
job["attempt"],
|
||||
JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS,
|
||||
)
|
||||
if row is None:
|
||||
LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"])
|
||||
return "skipped"
|
||||
|
||||
try:
|
||||
inject("after_claim")
|
||||
with tracer().start_as_current_span("attachment.read_staging"):
|
||||
content = store.objects.get(row["staging_key"], row["size_bytes"])
|
||||
if hashlib.sha256(content).hexdigest() != row["sha256"].strip():
|
||||
raise InvalidAttachment("checksum_mismatch")
|
||||
variants = _build_variants_bounded(
|
||||
attachment_id,
|
||||
content,
|
||||
job["pipeline_version"],
|
||||
JOB_TIMEOUT_SECONDS,
|
||||
)
|
||||
inject("before_original_put")
|
||||
with tracer().start_as_current_span("attachment.write_original"):
|
||||
store.objects.put(row["object_key"], content, row["content_type"], len(content))
|
||||
inject("after_original_put")
|
||||
persisted = []
|
||||
for variant in variants:
|
||||
data = variant.pop("content")
|
||||
inject(f"before_variant_put:{variant['name']}")
|
||||
with tracer().start_as_current_span("attachment.write_variant"):
|
||||
store.objects.put(variant["object_key"], data, variant["content_type"], len(data))
|
||||
inject(f"after_variant_put:{variant['name']}")
|
||||
persisted.append(variant)
|
||||
inject("before_commit")
|
||||
with tracer().start_as_current_span("attachment.commit_ready"):
|
||||
completed = store.complete_attachment_job(
|
||||
attachment_id,
|
||||
job["job_id"],
|
||||
job["pipeline_version"],
|
||||
job["attempt"],
|
||||
persisted,
|
||||
)
|
||||
if not completed:
|
||||
LOGGER.warning("job lease was superseded before completion: %s", job["job_id"])
|
||||
return "skipped"
|
||||
inject("after_commit")
|
||||
try:
|
||||
store.objects.delete(row["staging_key"])
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id)
|
||||
LOGGER.info("attachment processing completed: %s", attachment_id)
|
||||
return "success"
|
||||
except InvalidAttachment as exc:
|
||||
carrier = {}
|
||||
propagate.inject(carrier)
|
||||
store.fail_attachment_job(
|
||||
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
|
||||
MAX_ATTEMPTS, str(exc), None, carrier.get("traceparent"), carrier.get("tracestate"),
|
||||
)
|
||||
try:
|
||||
store.objects.delete(row["staging_key"])
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id)
|
||||
LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc)
|
||||
return "failed"
|
||||
except Exception as exc:
|
||||
delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25)
|
||||
carrier = {}
|
||||
propagate.inject(carrier)
|
||||
store.fail_attachment_job(
|
||||
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
|
||||
MAX_ATTEMPTS, "processing_failed", delay, carrier.get("traceparent"), carrier.get("tracestate"),
|
||||
)
|
||||
LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__)
|
||||
return "failed"
|
||||
|
||||
|
||||
def handle_delivery(
|
||||
store: Store,
|
||||
channel: pika.adapters.blocking_connection.BlockingChannel,
|
||||
method,
|
||||
body: bytes,
|
||||
fault_injector: Optional[Callable[[str], None]] = None,
|
||||
) -> None:
|
||||
try:
|
||||
job = _job(body)
|
||||
except (ValueError, TypeError, json.JSONDecodeError):
|
||||
LOGGER.exception("invalid broker job rejected")
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
|
||||
return
|
||||
parent_context = propagate.extract(
|
||||
{key: job[key] for key in ("traceparent", "tracestate") if key in job}
|
||||
)
|
||||
started = time.perf_counter()
|
||||
created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00"))
|
||||
with tracer().start_as_current_span(
|
||||
"attachment.process",
|
||||
context=parent_context,
|
||||
kind=SpanKind.CONSUMER,
|
||||
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "process"},
|
||||
) as span:
|
||||
worker_started()
|
||||
try:
|
||||
outcome = _process(store, job, fault_injector)
|
||||
if outcome == "failed":
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
if outcome != "skipped":
|
||||
elapsed = time.perf_counter() - started
|
||||
ready_duration = (
|
||||
max(0.0, datetime.now(timezone.utc).timestamp() - created_at.timestamp())
|
||||
if outcome == "success"
|
||||
else None
|
||||
)
|
||||
record_job(outcome, elapsed, ready_duration)
|
||||
finally:
|
||||
worker_stopped()
|
||||
if fault_injector is not None:
|
||||
fault_injector("before_ack")
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1:
|
||||
raise RuntimeError("Worker limits must be positive")
|
||||
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-worker"))
|
||||
store = Store()
|
||||
store.open()
|
||||
start_runtime_metrics(store, WORKER_CONCURRENCY, include_queue_snapshot=False)
|
||||
connection = None
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
|
||||
channel.basic_qos(prefetch_count=WORKER_CONCURRENCY)
|
||||
|
||||
def on_message(ch, method, properties, body):
|
||||
handle_delivery(store, ch, method, body)
|
||||
|
||||
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False)
|
||||
LOGGER.info("worker consuming durable attachment queue")
|
||||
channel.start_consuming()
|
||||
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
|
||||
LOGGER.exception("worker connection failed")
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
connection.close()
|
||||
connection = None
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202):
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={"Content-Type": content_type},
|
||||
schema="Attachment" if expected in (201, 202) else None,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
other_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
print("Checking accepted fixture upload.", flush=True)
|
||||
attachment = upload(client, alice, chat["id"], image)["json"]
|
||||
|
||||
print("Checking damaged and spoofed image rejection.", flush=True)
|
||||
upload(client, alice, chat["id"], b"not an image", expected=415)
|
||||
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
|
||||
print("Checking byte and pixel limits.", flush=True)
|
||||
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
def chunk(kind, payload):
|
||||
return (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
|
||||
bad_idat[-3] ^= 0xFF
|
||||
damaged_idat = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", bytes(bad_idat))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], damaged_idat, expected=415)
|
||||
|
||||
too_many_pixels = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(b""))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], too_many_pixels, expected=413)
|
||||
print("Checking filename and cross-chat authorization.", flush=True)
|
||||
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
|
||||
upload(client, eve, chat["id"], image, expected=404)
|
||||
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + other_chat["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
400,
|
||||
actor=bob,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": ""},
|
||||
)
|
||||
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a concurrent idempotency-key race through the load balancer."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Concurrent shared-idempotency integration check")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
text = "concurrent-" + uuid.uuid4().hex
|
||||
count = 12
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def send_once(_):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=10)
|
||||
return worker.message(alice, chat, text, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send_once, range(count)))
|
||||
|
||||
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
|
||||
messages = [response["json"] for response in responses]
|
||||
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
|
||||
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
matches = [message for message in page["items"] if message["text"] == text]
|
||||
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
|
||||
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify an upload committed while the publisher is stopped is processed after restart."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify committed attachment outbox intent survives publisher downtime")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
compose("stop", "publisher")
|
||||
try:
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not accepted into queued state")
|
||||
finally:
|
||||
compose("start", "publisher")
|
||||
|
||||
ready = client.wait_ready(alice, attachment)
|
||||
stored_original = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
thumbnail = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
variant = next(item for item in ready["variants"] if item["name"] == "thumbnail")
|
||||
check(stored_original == original, "recovered job changed original bytes")
|
||||
check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"accepted_status_before_publisher_restart": attachment["status"],
|
||||
"final_status": ready["status"],
|
||||
"attachment_id": attachment["id"],
|
||||
"original_sha256": hashlib.sha256(stored_original).hexdigest(),
|
||||
"thumbnail_sha256": variant["sha256"],
|
||||
"recovered_without_manual_republish": True,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import pika
|
||||
|
||||
from src.publisher import publish_pending
|
||||
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
|
||||
|
||||
|
||||
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
|
||||
JOB_ID = "00000000-0000-4000-8000-000000000002"
|
||||
|
||||
|
||||
class SimulatedCrash(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
def __init__(self, content):
|
||||
self.data = {"staging/" + ATTACHMENT_ID: content}
|
||||
self.puts = []
|
||||
|
||||
def get(self, key, max_bytes):
|
||||
return self.data[key]
|
||||
|
||||
def put(self, key, content, content_type, size_bytes=None):
|
||||
value = content if isinstance(content, bytes) else content.read()
|
||||
assert len(value) == size_bytes
|
||||
self.data[key] = value
|
||||
self.puts.append((key, hashlib.sha256(value).hexdigest()))
|
||||
|
||||
def delete(self, key):
|
||||
self.data.pop(key, None)
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self, content):
|
||||
self.objects = FakeObjects(content)
|
||||
self.row = {
|
||||
"id": ATTACHMENT_ID,
|
||||
"job_id": JOB_ID,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"status": "queued",
|
||||
"lease_until": 0.0,
|
||||
"staging_key": "staging/" + ATTACHMENT_ID,
|
||||
"object_key": "original/" + ATTACHMENT_ID,
|
||||
"content_type": "image/png",
|
||||
"size_bytes": len(content),
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"variants": [],
|
||||
}
|
||||
|
||||
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
|
||||
row = self.row
|
||||
if (
|
||||
attachment_id != row["id"]
|
||||
or job_id != row["job_id"]
|
||||
or version != row["pipeline_version"]
|
||||
or attempt != row["attempt"]
|
||||
or row["status"] in ("ready", "failed")
|
||||
):
|
||||
return None
|
||||
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
|
||||
return None
|
||||
row["status"] = "processing"
|
||||
row["lease_until"] = time.monotonic() + lease_seconds
|
||||
return dict(row)
|
||||
|
||||
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
|
||||
row = self.row
|
||||
if (
|
||||
attachment_id != row["id"]
|
||||
or job_id != row["job_id"]
|
||||
or version != row["pipeline_version"]
|
||||
or attempt != row["attempt"]
|
||||
or row["status"] != "processing"
|
||||
or row["lease_until"] <= time.monotonic()
|
||||
):
|
||||
return False
|
||||
row["status"] = "ready"
|
||||
row["lease_until"] = 0.0
|
||||
row["variants"] = variants
|
||||
return True
|
||||
|
||||
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
|
||||
row = self.row
|
||||
if (
|
||||
row["id"] == attachment_id
|
||||
and row["job_id"] == job_id
|
||||
and row["pipeline_version"] == version
|
||||
and row["attempt"] == attempt
|
||||
and row["status"] == "processing"
|
||||
):
|
||||
row["status"] = "failed" if delay is None else "queued"
|
||||
row["lease_until"] = 0.0
|
||||
row["error_code"] = error_code
|
||||
if delay is not None:
|
||||
row["attempt"] += 1
|
||||
|
||||
def recover_expired(self):
|
||||
if self.row["status"] == "processing":
|
||||
self.row["attempt"] += 1
|
||||
self.row["status"] = "queued"
|
||||
self.row["lease_until"] = 0.0
|
||||
|
||||
def replay(self, version):
|
||||
row = self.row
|
||||
previous_job = row["job_id"]
|
||||
row["job_id"] = str(uuid.uuid4())
|
||||
row["pipeline_version"] = version
|
||||
row["attempt"] = 1
|
||||
row["status"] = "queued"
|
||||
row["lease_until"] = 0.0
|
||||
row["variants"] = []
|
||||
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
|
||||
self.objects.put(
|
||||
row["staging_key"],
|
||||
self.objects.data[row["object_key"]],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": row["job_id"],
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": version,
|
||||
"attempt": 1,
|
||||
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
"previous_job_id": previous_job,
|
||||
}
|
||||
|
||||
|
||||
class FakeChannel:
|
||||
def __init__(self):
|
||||
self.acked = []
|
||||
self.nacked = []
|
||||
|
||||
def basic_ack(self, delivery_tag):
|
||||
self.acked.append(delivery_tag)
|
||||
|
||||
def basic_nack(self, delivery_tag, requeue):
|
||||
self.nacked.append((delivery_tag, requeue))
|
||||
|
||||
|
||||
class FakeMethod:
|
||||
def __init__(self, delivery_tag):
|
||||
self.delivery_tag = delivery_tag
|
||||
|
||||
|
||||
def job_body(job_id=JOB_ID, version=1, attempt=1):
|
||||
return json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"job_id": job_id,
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": version,
|
||||
"attempt": attempt,
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def crash_at(expected):
|
||||
def inject(point):
|
||||
if point == expected:
|
||||
raise SimulatedCrash(point)
|
||||
|
||||
return inject
|
||||
|
||||
|
||||
def slow_builder(attachment_id, content, pipeline_version):
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def test_duplicate_and_ack_loss(content):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
try:
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(1), job_body(),
|
||||
fault_injector=crash_at("before_ack"),
|
||||
)
|
||||
raise AssertionError("fault injection did not crash before ack")
|
||||
except SimulatedCrash:
|
||||
pass
|
||||
assert store.row["status"] == "ready"
|
||||
first_write_count = len(store.objects.puts)
|
||||
assert channel.acked == []
|
||||
|
||||
handle_delivery(store, channel, FakeMethod(2), job_body())
|
||||
assert channel.acked == [2]
|
||||
assert len(store.objects.puts) == first_write_count
|
||||
assert store.row["status"] == "ready"
|
||||
return {
|
||||
"scenario": "duplicate_delivery_after_commit_before_ack",
|
||||
"deliveries": 2,
|
||||
"effective_object_writes": first_write_count,
|
||||
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
|
||||
}
|
||||
|
||||
|
||||
def test_crash_before_and_after_side_effect(content, crash_point):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
try:
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(3), job_body(),
|
||||
fault_injector=crash_at(crash_point),
|
||||
)
|
||||
raise AssertionError(f"fault injection did not crash at {crash_point}")
|
||||
except SimulatedCrash:
|
||||
pass
|
||||
assert channel.acked == []
|
||||
store.recover_expired()
|
||||
retry = job_body(attempt=2)
|
||||
handle_delivery(store, channel, FakeMethod(4), retry)
|
||||
assert store.row["status"] == "ready"
|
||||
assert store.row["attempt"] == 2
|
||||
assert channel.acked == [4]
|
||||
assert store.objects.data["original/" + ATTACHMENT_ID] == content
|
||||
return {
|
||||
"scenario": "crash_" + crash_point,
|
||||
"attempts": store.row["attempt"],
|
||||
"terminal_status": store.row["status"],
|
||||
}
|
||||
|
||||
|
||||
def test_reordered_attempts(content):
|
||||
store = FakeStore(content)
|
||||
store.row["attempt"] = 2
|
||||
channel = FakeChannel()
|
||||
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
|
||||
writes_after_current = len(store.objects.puts)
|
||||
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
|
||||
assert store.row["status"] == "ready"
|
||||
assert len(store.objects.puts) == writes_after_current
|
||||
assert channel.acked == [5, 6]
|
||||
return {
|
||||
"scenario": "out_of_order_attempt_delivery",
|
||||
"current_attempt": store.row["attempt"],
|
||||
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
|
||||
}
|
||||
|
||||
|
||||
def test_stale_pipeline_cannot_publish_newer_result(content):
|
||||
store = FakeStore(content)
|
||||
channel = FakeChannel()
|
||||
replayed = []
|
||||
|
||||
def supersede_after_first_variant(point):
|
||||
if point == "after_variant_put:thumbnail" and not replayed:
|
||||
replayed.append(store.replay(2))
|
||||
|
||||
handle_delivery(
|
||||
store, channel, FakeMethod(7), job_body(),
|
||||
fault_injector=supersede_after_first_variant,
|
||||
)
|
||||
assert store.row["status"] == "queued"
|
||||
assert store.row["pipeline_version"] == 2
|
||||
assert channel.acked == [7]
|
||||
|
||||
current_job = replayed[0]
|
||||
handle_delivery(
|
||||
store,
|
||||
channel,
|
||||
FakeMethod(8),
|
||||
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
|
||||
)
|
||||
assert store.row["status"] == "ready"
|
||||
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
|
||||
assert channel.acked == [7, 8]
|
||||
return {
|
||||
"scenario": "stale_pipeline_after_newer_version",
|
||||
"current_pipeline_version": store.row["pipeline_version"],
|
||||
"published_variant_versions": sorted(
|
||||
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class FakeTransaction:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def __enter__(self):
|
||||
self.published = False
|
||||
self.store.current_transaction = self
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
if exc_type is None and self.published:
|
||||
self.store.outbox_published = True
|
||||
self.store.current_transaction = None
|
||||
return False
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def transaction(self):
|
||||
return FakeTransaction(self.store)
|
||||
|
||||
def execute(self, query, params):
|
||||
assert "UPDATE attachment_outbox" in query
|
||||
assert params == ("outbox-row",)
|
||||
self.store.current_transaction.published = True
|
||||
|
||||
|
||||
class FakePool:
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
|
||||
def connection(self):
|
||||
connection = FakeConnection(self.store)
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self_inner):
|
||||
return connection
|
||||
|
||||
def __exit__(self_inner, exc_type, exc_value, traceback):
|
||||
return False
|
||||
|
||||
return ConnectionContext()
|
||||
|
||||
|
||||
class FakeOutboxStore:
|
||||
def __init__(self):
|
||||
self.pool = FakePool(self)
|
||||
self.outbox_published = False
|
||||
self.current_transaction = None
|
||||
|
||||
def claim_outbox(self, connection):
|
||||
if self.outbox_published:
|
||||
return None
|
||||
return {
|
||||
"id": "outbox-row",
|
||||
"job_id": JOB_ID,
|
||||
"attachment_id": ATTACHMENT_ID,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
class AmbiguousConfirmChannel:
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
self.fail_once = True
|
||||
|
||||
def basic_publish(self, **message):
|
||||
self.messages.append(message)
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
|
||||
|
||||
|
||||
class FailBeforePublishChannel:
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
self.fail_once = True
|
||||
|
||||
def basic_publish(self, **message):
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
|
||||
self.messages.append(message)
|
||||
|
||||
|
||||
def test_committed_outbox_survives_publisher_crash():
|
||||
store = FakeOutboxStore()
|
||||
channel = FailBeforePublishChannel()
|
||||
try:
|
||||
publish_pending(store, channel)
|
||||
raise AssertionError("publisher crash was not injected")
|
||||
except pika.exceptions.AMQPConnectionError:
|
||||
pass
|
||||
assert not store.outbox_published
|
||||
assert channel.messages == []
|
||||
assert publish_pending(store, channel)
|
||||
assert store.outbox_published
|
||||
assert len(channel.messages) == 1
|
||||
return {
|
||||
"scenario": "publisher_crash_before_publish",
|
||||
"published_after_restart": len(channel.messages),
|
||||
"outbox_recovered_after_restart": store.outbox_published,
|
||||
}
|
||||
|
||||
|
||||
def test_outbox_confirm_gap_republishes_same_job():
|
||||
store = FakeOutboxStore()
|
||||
channel = AmbiguousConfirmChannel()
|
||||
try:
|
||||
publish_pending(store, channel)
|
||||
raise AssertionError("publisher confirm fault was not injected")
|
||||
except pika.exceptions.AMQPConnectionError:
|
||||
pass
|
||||
assert not store.outbox_published
|
||||
assert publish_pending(store, channel)
|
||||
assert store.outbox_published
|
||||
assert len(channel.messages) == 2
|
||||
first, second = channel.messages
|
||||
assert first["body"] == second["body"]
|
||||
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
|
||||
return {
|
||||
"scenario": "publish_confirm_lost_after_broker_accept",
|
||||
"deliveries": len(channel.messages),
|
||||
"same_message_id": True,
|
||||
}
|
||||
|
||||
|
||||
def test_image_processing_has_a_hard_timeout():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
_build_variants_bounded(
|
||||
ATTACHMENT_ID,
|
||||
b"bounded timeout test",
|
||||
1,
|
||||
timeout_seconds=1,
|
||||
builder=slow_builder,
|
||||
)
|
||||
raise AssertionError("slow transform exceeded its time budget without termination")
|
||||
except ProcessingTimeout:
|
||||
pass
|
||||
elapsed = time.monotonic() - started
|
||||
assert elapsed < 4
|
||||
return {
|
||||
"scenario": "transform_timeout_terminated",
|
||||
"configured_timeout_seconds": 1,
|
||||
"observed_seconds": round(elapsed, 3),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
results = [
|
||||
test_duplicate_and_ack_loss(content),
|
||||
test_crash_before_and_after_side_effect(content, "after_claim"),
|
||||
test_crash_before_and_after_side_effect(content, "after_original_put"),
|
||||
test_reordered_attempts(content),
|
||||
test_stale_pipeline_cannot_publish_newer_result(content),
|
||||
test_committed_outbox_survives_publisher_crash(),
|
||||
test_outbox_confirm_gap_republishes_same_job(),
|
||||
test_image_processing_has_a_hard_timeout(),
|
||||
]
|
||||
report = {
|
||||
"result": "PASS",
|
||||
"track": "2",
|
||||
"fixture": "tests/fixtures/sample.png",
|
||||
"scenarios": results,
|
||||
"invariants": {
|
||||
"confirmed_work_eventually_ready": True,
|
||||
"duplicate_effects_create_no_second_variant_set": True,
|
||||
"stale_attempt_writes_no_effects": True,
|
||||
"stale_pipeline_cannot_publish_current_metadata": True,
|
||||
"ambiguous_confirm_republishes_stable_job": True,
|
||||
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
|
||||
},
|
||||
}
|
||||
evidence = ROOT / "evidence" / "pipeline-faults.json"
|
||||
evidence.parent.mkdir(exist_ok=True)
|
||||
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, sort_keys=True))
|
||||
print(f"Evidence written to {evidence.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify explicit replay changes the deployed pipeline while preserving the original."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
MARK_FAILED_CODE = """
|
||||
import sys
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
with store.pool.connection() as conn:
|
||||
updated = conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', "
|
||||
"variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'",
|
||||
(sys.argv[1],),
|
||||
)
|
||||
if updated.rowcount != 1:
|
||||
raise RuntimeError("test attachment was not ready for failed replay injection")
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
AUDIT_CODE = """
|
||||
import json
|
||||
import sys
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
with store.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events "
|
||||
"WHERE attachment_id = %s ORDER BY id",
|
||||
(sys.argv[1],),
|
||||
).fetchall()
|
||||
print(json.dumps(rows, sort_keys=True))
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
|
||||
def docker_exec(container_id, *args):
|
||||
return subprocess.run(
|
||||
["docker", "exec", container_id, *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def replay(container_id, attachment_id, version):
|
||||
result = docker_exec(
|
||||
container_id,
|
||||
"python3",
|
||||
"-m",
|
||||
"src.replay_attachment",
|
||||
"--attachment-id",
|
||||
attachment_id,
|
||||
"--pipeline-version",
|
||||
str(version),
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify versioned attachment replay against the running Compose stack")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
ready_v1 = client.wait_ready(alice, attachment)
|
||||
v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail")
|
||||
v1_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch")
|
||||
|
||||
containers = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.split()
|
||||
check(bool(containers), "no running API container found")
|
||||
replay_v2 = replay(containers[0], attachment["id"], 2)
|
||||
check(replay_v2["status"] == "queued", "replay was not queued")
|
||||
check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2")
|
||||
check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id")
|
||||
|
||||
ready_v2 = client.wait_ready(alice, ready_v1)
|
||||
v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail")
|
||||
v2_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
original_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=alice,
|
||||
)["bytes"]
|
||||
check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding")
|
||||
check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch")
|
||||
check(original_bytes == original, "pipeline replay changed the original file")
|
||||
|
||||
docker_exec(
|
||||
containers[0],
|
||||
"python3",
|
||||
"-c",
|
||||
MARK_FAILED_CODE,
|
||||
attachment["id"],
|
||||
)
|
||||
failed_replay = replay(containers[0], attachment["id"], 2)
|
||||
check(failed_replay["status"] == "queued", "failed attachment replay was not queued")
|
||||
check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version")
|
||||
check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id")
|
||||
ready_replayed = client.wait_ready(alice, ready_v2)
|
||||
audit = json.loads(
|
||||
docker_exec(
|
||||
containers[0],
|
||||
"python3",
|
||||
"-c",
|
||||
AUDIT_CODE,
|
||||
attachment["id"],
|
||||
).stdout
|
||||
)
|
||||
check(
|
||||
[(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit]
|
||||
== [(1, 2, "replay"), (2, 2, "replay")],
|
||||
"pipeline replay audit trail is incomplete",
|
||||
)
|
||||
check(ready_replayed["status"] == "ready", "failed attachment did not recover")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"previous_job_id": replay_v2["previous_job_id"],
|
||||
"job_id": replay_v2["job_id"],
|
||||
"pipeline_version": replay_v2["pipeline_version"],
|
||||
"failed_replay_job_id": failed_replay["job_id"],
|
||||
"audited_replays": len(audit),
|
||||
"v1_thumbnail_sha256": v1_thumbnail["sha256"],
|
||||
"v2_thumbnail_sha256": v2_thumbnail["sha256"],
|
||||
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
|
||||
import os
|
||||
import hashlib
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from src.reconcile import reconcile
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
def __init__(self, items):
|
||||
self.items = items
|
||||
self.deleted = []
|
||||
|
||||
def objects(self):
|
||||
return iter(self.items)
|
||||
|
||||
def delete(self, key):
|
||||
self.deleted.append(key)
|
||||
self.items = [(name, date) for name, date in self.items if name != key]
|
||||
|
||||
def put(self, key, content, content_type):
|
||||
self.items.append((key, datetime.now(timezone.utc)))
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self):
|
||||
now = datetime.now(timezone.utc)
|
||||
self.metadata = {
|
||||
"original/present": {"id": "present-id", "object_key": "original/present"},
|
||||
"original/missing": {
|
||||
"id": "missing-id",
|
||||
"object_key": "original/missing",
|
||||
"size_bytes": 4,
|
||||
"sha256": hashlib.sha256(b"test").hexdigest(),
|
||||
"content_type": "image/png",
|
||||
},
|
||||
}
|
||||
self.objects = FakeObjects(
|
||||
[
|
||||
("original/present", now),
|
||||
("original/stale-orphan", now - timedelta(hours=2)),
|
||||
("original/new-orphan", now - timedelta(minutes=10)),
|
||||
]
|
||||
)
|
||||
self.missing_incidents = set()
|
||||
self.cleared_incidents = set()
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
return self.metadata
|
||||
|
||||
def record_missing_object(self, attachment_id):
|
||||
self.missing_incidents.add(attachment_id)
|
||||
|
||||
def clear_missing_object(self, attachment_id):
|
||||
self.missing_incidents.discard(attachment_id)
|
||||
self.cleared_incidents.add(attachment_id)
|
||||
|
||||
|
||||
def main():
|
||||
store = FakeStore()
|
||||
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
|
||||
assert dry_run["stale_orphan_candidates"] == 1
|
||||
assert dry_run["missing_objects"] == 1
|
||||
assert dry_run["deleted_orphans"] == 0
|
||||
assert store.objects.deleted == [] and store.missing_incidents == set()
|
||||
|
||||
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert "original/new-orphan" not in store.objects.deleted
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
assert store.cleared_incidents == {"present-id"}
|
||||
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
restore_dir = Path(backup)
|
||||
(restore_dir / "missing-id").write_bytes(b"test")
|
||||
preview = reconcile(store, 3600, False, restore_dir)
|
||||
assert preview["restorable_missing_objects"] == 1
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
restored = reconcile(store, 3600, True, restore_dir)
|
||||
assert restored["restored_missing_objects"] == 1
|
||||
assert store.missing_incidents == set()
|
||||
|
||||
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert store.missing_incidents == set()
|
||||
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
STORE_CODE = """
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
action = sys.argv[1]
|
||||
if action == "key":
|
||||
attachment_id = sys.argv[2]
|
||||
matches = [
|
||||
key for key, row in store.reconciliation_snapshot().items()
|
||||
if str(row["id"]) == attachment_id and row["reconcile_type"] == "original"
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("attachment metadata lookup failed")
|
||||
print(matches[0])
|
||||
elif action == "put":
|
||||
key, file_path = sys.argv[2:4]
|
||||
content = Path(file_path).read_bytes()
|
||||
store.objects.put(key, content, "image/png", len(content))
|
||||
elif action == "delete":
|
||||
store.objects.delete(sys.argv[2])
|
||||
elif action == "exists":
|
||||
print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no")
|
||||
else:
|
||||
raise RuntimeError("unsupported test operation")
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
PRIVATE_CHECK_CODE = """
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
socket.getaddrinfo("s3-core", 8888)
|
||||
except socket.gaierror:
|
||||
print("PASS: SeaweedFS master/filer network is isolated")
|
||||
else:
|
||||
raise RuntimeError("API network can resolve the SeaweedFS core service")
|
||||
|
||||
for port in (8181, 9101):
|
||||
try:
|
||||
socket.create_connection(("s3", port), timeout=1).close()
|
||||
except OSError:
|
||||
continue
|
||||
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
|
||||
|
||||
try:
|
||||
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 403:
|
||||
print("PASS: unauthenticated bucket access denied")
|
||||
else:
|
||||
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
|
||||
else:
|
||||
raise RuntimeError("private bucket allowed an unauthenticated GET")
|
||||
"""
|
||||
|
||||
|
||||
def docker_exec(container_id, *args):
|
||||
return subprocess.run(
|
||||
["docker", "exec", container_id, *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def reconciliation(container_id, *args):
|
||||
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def object_exists(container_id, key):
|
||||
return docker_exec(
|
||||
container_id, "python3", "-c", STORE_CODE, "exists", key
|
||||
).stdout.strip() == "yes"
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = result.stdout.split()
|
||||
check(len(containers) >= 2, "at least two running API replicas are required")
|
||||
container_id = containers[0]
|
||||
key = docker_exec(
|
||||
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
|
||||
).stdout.strip()
|
||||
orphan_key = "original/" + str(uuid.uuid4())
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
source = Path(backup) / "original.png"
|
||||
source.write_bytes(content)
|
||||
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
|
||||
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
|
||||
recovery_file = recovery_dir + "/" + attachment["id"]
|
||||
orphan_created = False
|
||||
orphan_removed = False
|
||||
missing_deleted = False
|
||||
missing_restored = False
|
||||
try:
|
||||
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
|
||||
orphan_created = True
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
|
||||
missing_deleted = True
|
||||
|
||||
preview = reconciliation(container_id)
|
||||
check(
|
||||
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
|
||||
"dry-run missed injected inconsistencies",
|
||||
)
|
||||
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
|
||||
immediate_apply = reconciliation(container_id, "--apply")
|
||||
check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period")
|
||||
|
||||
docker_exec(container_id, "mkdir", "-p", recovery_dir)
|
||||
subprocess.run(
|
||||
["docker", "cp", str(source), container_id + ":" + recovery_file],
|
||||
check=True,
|
||||
)
|
||||
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
|
||||
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
|
||||
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
|
||||
missing_restored = True
|
||||
|
||||
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
|
||||
check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period")
|
||||
print("Waiting for the 60-second orphan grace period.")
|
||||
time.sleep(61)
|
||||
|
||||
cleanup_preview = reconciliation(container_id)
|
||||
check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace")
|
||||
cleaned = reconciliation(container_id, "--apply")
|
||||
orphan_removed = not object_exists(container_id, orphan_key)
|
||||
check(orphan_removed, "injected stale orphan was not removed")
|
||||
final_scan = reconciliation(container_id, "--apply")
|
||||
check(
|
||||
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
|
||||
"repeat scan is not clean",
|
||||
)
|
||||
finally:
|
||||
if missing_deleted and not missing_restored:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
|
||||
if orphan_created and not orphan_removed:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
|
||||
|
||||
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
|
||||
check("PASS:" in private, "unauthenticated S3 access check failed")
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
|
||||
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"dataset_objects_scanned": final_scan["objects_seen"],
|
||||
"metadata_records_scanned": final_scan["metadata_seen"],
|
||||
"scan_duration_ms": final_scan["duration_ms"],
|
||||
"orphan_detected_and_removed": cleaned["deleted_orphans"],
|
||||
"missing_detected_and_restored": restored["restored_missing_objects"],
|
||||
"fresh_orphan_preserved_until_grace": True,
|
||||
"message_attachment_download_checksum": "unchanged",
|
||||
"private_bucket": "anonymous request denied",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local transformation checks for the lab5 image worker."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret")
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from src.worker import build_variants
|
||||
|
||||
|
||||
def main():
|
||||
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
attachment_id = "00000000-0000-4000-8000-000000000001"
|
||||
variants = build_variants(attachment_id, original, pipeline_version=1)
|
||||
by_name = {item["name"]: item for item in variants}
|
||||
assert set(by_name) == {"thumbnail", "crop", "optimized"}
|
||||
assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160)
|
||||
assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128)
|
||||
assert by_name["optimized"]["content_type"] == "image/webp"
|
||||
for item in variants:
|
||||
content = item["content"]
|
||||
assert len(content) == item["size_bytes"]
|
||||
assert hashlib.sha256(content).hexdigest() == item["sha256"]
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
assert image.size == (item["width"], item["height"])
|
||||
newer = build_variants(attachment_id, original, pipeline_version=2)
|
||||
assert all(f"/v2/" in item["object_key"] for item in newer)
|
||||
assert all(f"/v1/" in item["object_key"] for item in variants)
|
||||
assert by_name["thumbnail"]["sha256"] != {
|
||||
item["name"]: item for item in newer
|
||||
}["thumbnail"]["sha256"]
|
||||
print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify queued uploads and message links survive an API/worker restart."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify queued attachment recovery after API and worker restart")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
compose("stop", "worker")
|
||||
worker_stopped = True
|
||||
try:
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not accepted while worker was stopped")
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message")
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
409,
|
||||
actor=bob,
|
||||
)
|
||||
compose("restart", "api")
|
||||
compose("start", "worker")
|
||||
worker_stopped = False
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
original_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(original_bytes == original, "recovered worker changed original bytes")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
recovered_message = next(item for item in page["items"] if item["id"] == message["id"])
|
||||
check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"initial_status": attachment["status"],
|
||||
"final_status": ready["status"],
|
||||
"message_attachment_link_survived": True,
|
||||
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if worker_stopped:
|
||||
compose("start", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeat a paced synthetic message workload with telemetry off and sampled."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
|
||||
def command(args, capture=True):
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.PIPE if capture else None,
|
||||
)
|
||||
return result.stdout.strip() if capture else ""
|
||||
|
||||
|
||||
def set_env_value(text, name, value):
|
||||
lines = text.splitlines()
|
||||
prefix = name + "="
|
||||
for index, line in enumerate(lines):
|
||||
if line.startswith(prefix):
|
||||
lines[index] = prefix + value
|
||||
break
|
||||
else:
|
||||
lines.append(prefix + value)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def parse_env(text):
|
||||
values = {}
|
||||
for line in text.splitlines():
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
name, value = line.split("=", 1)
|
||||
values[name] = value
|
||||
return values
|
||||
|
||||
|
||||
def wait_ready(url, ca_file=None, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
context = ssl.create_default_context(cafile=ca_file)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url + "/health/ready", context=context, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
time.sleep(2)
|
||||
raise RuntimeError("API did not become ready after the sampling mode change")
|
||||
|
||||
|
||||
def parse_size(value):
|
||||
match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value)
|
||||
if not match:
|
||||
return None
|
||||
factors = {
|
||||
"B": 1,
|
||||
"kB": 1000,
|
||||
"MB": 1000**2,
|
||||
"GB": 1000**3,
|
||||
"KiB": 1024,
|
||||
"MiB": 1024**2,
|
||||
"GiB": 1024**3,
|
||||
}
|
||||
return int(float(match.group(1)) * factors[match.group(2)])
|
||||
|
||||
|
||||
def docker_resources():
|
||||
rows = command(
|
||||
[
|
||||
"docker",
|
||||
"stats",
|
||||
"--no-stream",
|
||||
"--format",
|
||||
"{{.Name}};{{.CPUPerc}};{{.MemUsage}}",
|
||||
]
|
||||
)
|
||||
cpu_total = 0.0
|
||||
memory_total = 0
|
||||
found = False
|
||||
for row in rows.splitlines():
|
||||
fields = row.split(";")
|
||||
if len(fields) != 3 or not any(
|
||||
name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-")
|
||||
):
|
||||
continue
|
||||
cpu = re.match(r"\s*([\d.]+)%", fields[1])
|
||||
memory = parse_size(fields[2].split("/", 1)[0])
|
||||
if cpu and memory is not None:
|
||||
cpu_total += float(cpu.group(1))
|
||||
memory_total += memory
|
||||
found = True
|
||||
return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None
|
||||
|
||||
|
||||
def storage_sizes(compose):
|
||||
paths = {
|
||||
"prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]),
|
||||
"loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]),
|
||||
"tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]),
|
||||
}
|
||||
result = {}
|
||||
for name, args in paths.items():
|
||||
output = command(compose + args)
|
||||
result[name] = int(output.split()[0])
|
||||
return result
|
||||
|
||||
|
||||
def tempo_message_traces(compose, start, end):
|
||||
url = (
|
||||
"http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000"
|
||||
"&tags=service.name%%3Dmessenger-api" % (start, end)
|
||||
)
|
||||
output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url])
|
||||
data = json.loads(output)
|
||||
return [
|
||||
item
|
||||
for item in data.get("traces", [])
|
||||
if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"
|
||||
]
|
||||
|
||||
|
||||
def mean_spans(compose, traces, sample_limit=20):
|
||||
counts = []
|
||||
for item in traces[:sample_limit]:
|
||||
output = command(
|
||||
compose
|
||||
+ [
|
||||
"exec",
|
||||
"-T",
|
||||
"tempo",
|
||||
"wget",
|
||||
"-qO-",
|
||||
"http://127.0.0.1:3200/api/traces/" + item["traceID"],
|
||||
]
|
||||
)
|
||||
data = json.loads(output)
|
||||
count = sum(
|
||||
len(scope.get("spans", []))
|
||||
for batch in data.get("batches", [])
|
||||
for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", []))
|
||||
)
|
||||
counts.append(count)
|
||||
return round(sum(counts) / len(counts), 2) if counts else 0.0
|
||||
|
||||
|
||||
def run_load(base_url, ca_file, duration, concurrency, target_rps, alice, chat):
|
||||
from client import Client, Failure
|
||||
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
ticket_lock = threading.Lock()
|
||||
ticket = 0
|
||||
latencies = []
|
||||
failures = 0
|
||||
results_lock = threading.Lock()
|
||||
start_clock = time.monotonic()
|
||||
start_epoch = int(time.time())
|
||||
deadline = start_clock + duration
|
||||
|
||||
def worker(worker_id):
|
||||
nonlocal ticket, failures
|
||||
client = Client(base_url, ca_file, timeout=5)
|
||||
local_latencies = []
|
||||
local_failures = 0
|
||||
barrier.wait()
|
||||
while True:
|
||||
with ticket_lock:
|
||||
current = ticket
|
||||
ticket += 1
|
||||
due = start_clock + current / target_rps
|
||||
if due >= deadline:
|
||||
break
|
||||
delay = due - time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current))
|
||||
local_latencies.append(time.monotonic() - started)
|
||||
except Failure:
|
||||
local_failures += 1
|
||||
with results_lock:
|
||||
latencies.extend(local_latencies)
|
||||
failures += local_failures
|
||||
|
||||
samples = []
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)]
|
||||
barrier.wait()
|
||||
while any(not future.done() for future in futures):
|
||||
sample = docker_resources()
|
||||
if sample is not None:
|
||||
samples.append(sample)
|
||||
time.sleep(2)
|
||||
for future in futures:
|
||||
future.result()
|
||||
elapsed = time.monotonic() - start_clock
|
||||
sorted_latencies = sorted(latencies)
|
||||
|
||||
def percentile(value):
|
||||
if not sorted_latencies:
|
||||
return None
|
||||
index = max(0, math.ceil(value * len(sorted_latencies)) - 1)
|
||||
return round(sorted_latencies[index] * 1000, 3)
|
||||
|
||||
total = len(sorted_latencies) + failures
|
||||
report = {
|
||||
"workload": "paced synthetic POST messages; no automatic retries",
|
||||
"requested_duration_seconds": duration,
|
||||
"target_rps": target_rps,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"concurrency": concurrency,
|
||||
"successful_requests": len(sorted_latencies),
|
||||
"failed_requests": failures,
|
||||
"error_ratio": round(failures / total, 6) if total else None,
|
||||
"successful_rps": round(len(sorted_latencies) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(0.50),
|
||||
"p95": percentile(0.95),
|
||||
"p99": percentile(0.99),
|
||||
},
|
||||
"docker_resource_samples": len(samples),
|
||||
}
|
||||
if samples:
|
||||
report["docker_resources"] = {
|
||||
"mean_cpu_percent_sum": round(
|
||||
sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2
|
||||
),
|
||||
"peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples),
|
||||
"peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples),
|
||||
"services": ["api", "worker", "publisher", "otel-collector"],
|
||||
}
|
||||
return report, start_epoch, int(time.time())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--ca-file", default=None)
|
||||
parser.add_argument("--duration", type=int, default=15)
|
||||
parser.add_argument("--concurrency", type=int, default=2)
|
||||
parser.add_argument("--target-rps", type=int, default=20)
|
||||
parser.add_argument("--warmup-seconds", type=int, default=5)
|
||||
parser.add_argument("--output", default="evidence/track1-observability-cost.json")
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1:
|
||||
parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive")
|
||||
|
||||
env_path = ROOT / args.env_file
|
||||
if not env_path.is_file():
|
||||
parser.error("create the local .env from .env.example and generate secrets first")
|
||||
original = env_path.read_text(encoding="utf-8")
|
||||
values = parse_env(original)
|
||||
base_url = args.base_url or "https://localhost:" + values.get("APP_PORT", "8443")
|
||||
compose = ["docker", "compose", "--env-file", str(env_path)]
|
||||
output_path = ROOT / args.output
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
results = []
|
||||
configurations = [
|
||||
("off", "true", "1.0"),
|
||||
("sample-100-percent", "false", "1.0"),
|
||||
("sample-25-percent", "false", "0.25"),
|
||||
("sample-10-percent", "false", "0.1"),
|
||||
]
|
||||
try:
|
||||
for name, disabled, ratio in configurations:
|
||||
changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled)
|
||||
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
|
||||
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio)
|
||||
env_path.write_text(changed, encoding="utf-8")
|
||||
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
|
||||
wait_ready(base_url, args.ca_file)
|
||||
time.sleep(args.warmup_seconds)
|
||||
from client import Client
|
||||
|
||||
alice, _, _, chat = Client(base_url, args.ca_file, timeout=10).bootstrap()
|
||||
time.sleep(2)
|
||||
before = storage_sizes(compose)
|
||||
load, start, end = run_load(
|
||||
base_url, args.ca_file, args.duration, args.concurrency, args.target_rps, alice, chat
|
||||
)
|
||||
time.sleep(15)
|
||||
after = storage_sizes(compose)
|
||||
traces = tempo_message_traces(compose, start - 5, end + 20)
|
||||
spans_per_trace = mean_spans(compose, traces)
|
||||
results.append(
|
||||
{
|
||||
"configuration": name,
|
||||
"sdk_disabled": disabled == "true",
|
||||
"trace_sampling_ratio": float(ratio),
|
||||
"load": load,
|
||||
"window_unix_seconds": {"start": start, "end": end},
|
||||
"sampled_message_trace_count": len(traces),
|
||||
"estimated_spans_per_message_trace": spans_per_trace,
|
||||
"persistent_storage_bytes_delta": {
|
||||
signal: after[signal] - before[signal] for signal in before
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
baseline = next(
|
||||
item["estimated_spans_per_message_trace"]
|
||||
for item in results
|
||||
if item["configuration"] == "sample-100-percent"
|
||||
)
|
||||
for item in results:
|
||||
expected_spans = item["load"]["successful_requests"] * baseline
|
||||
observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"]
|
||||
item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2)
|
||||
|
||||
report = {
|
||||
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
|
||||
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
|
||||
"method": {
|
||||
"duration_seconds": args.duration,
|
||||
"concurrency": args.concurrency,
|
||||
"target_rps": args.target_rps,
|
||||
"warmup_seconds": args.warmup_seconds,
|
||||
"telemetry_flush_wait_seconds": 15,
|
||||
"hardware": {
|
||||
"platform": platform.platform(),
|
||||
"processor": platform.processor() or "unspecified",
|
||||
"logical_cpus": os.cpu_count(),
|
||||
},
|
||||
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
|
||||
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.",
|
||||
},
|
||||
"limitations": [
|
||||
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
|
||||
"Database history grows between modes; all modes use identical duration and concurrency.",
|
||||
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
|
||||
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
|
||||
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.",
|
||||
],
|
||||
"results": results,
|
||||
"balanced_configuration": "sample-25-percent",
|
||||
}
|
||||
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
env_path.write_text(original, encoding="utf-8")
|
||||
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
|
||||
|
||||
|
||||
def usage_bytes(line):
|
||||
match = USAGE_RE.search(line)
|
||||
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
|
||||
|
||||
|
||||
def api_containers():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
ids = result.stdout.split()
|
||||
check(len(ids) >= 2, "at least two running API containers are required")
|
||||
return ids
|
||||
|
||||
|
||||
def sample_once(ids):
|
||||
output = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.splitlines()
|
||||
values = [usage_bytes(line) for line in output]
|
||||
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
|
||||
return max(values)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Bounded-memory ten-client upload experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
width = height = 1400
|
||||
content = BytesIO()
|
||||
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
|
||||
content, format="PNG", compress_level=0
|
||||
)
|
||||
payload = content.getvalue()
|
||||
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
|
||||
ids = api_containers()
|
||||
baseline = sample_once(ids)
|
||||
samples = []
|
||||
stats_process = subprocess.Popen(
|
||||
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
def sampler():
|
||||
if stats_process.stdout is None:
|
||||
return
|
||||
for line in stats_process.stdout:
|
||||
value = usage_bytes(line)
|
||||
if value:
|
||||
samples.append(value)
|
||||
|
||||
sampling_thread = threading.Thread(target=sampler, daemon=True)
|
||||
sampling_thread.start()
|
||||
count = 10
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def upload(index):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=15)
|
||||
return worker.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
|
||||
202,
|
||||
actor=alice,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(upload, range(count)))
|
||||
finally:
|
||||
stopped_by_test = False
|
||||
if stats_process.poll() is None:
|
||||
stats_process.kill()
|
||||
stopped_by_test = True
|
||||
try:
|
||||
stats_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
stats_process.kill()
|
||||
stats_process.wait(timeout=5)
|
||||
sampling_thread.join(timeout=5)
|
||||
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
|
||||
check(
|
||||
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
|
||||
"docker stats sampling process failed (exit "
|
||||
+ str(stats_process.returncode)
|
||||
+ "): "
|
||||
+ stats_error.strip()[:300],
|
||||
)
|
||||
check(bool(samples), "docker stats produced no memory samples")
|
||||
peak = max([baseline] + samples)
|
||||
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"uploads": count,
|
||||
"concurrency": count,
|
||||
"bytes_per_upload": len(payload),
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"api_containers": len(ids),
|
||||
"sample_count": len(samples) // len(ids),
|
||||
"baseline_max_container_bytes": baseline,
|
||||
"peak_max_container_bytes": peak,
|
||||
"peak_delta_bytes": max(0, peak - baseline),
|
||||
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
|
||||
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
+25
-12
@@ -1,12 +1,21 @@
|
||||
# Публичные проверки
|
||||
# Проверки lab7
|
||||
|
||||
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
|
||||
HTTP проверки выполняются против запущенного Compose стенда; они не создают его автоматически. Клиенты Python используют стандартную библиотеку; сервер и unit tests требуют runtime зависимостей из `requirements.txt`. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://localhost:8080
|
||||
# без make:
|
||||
python3 tests/smoke.py --base-url http://localhost:8080
|
||||
make test BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-security BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-primitives
|
||||
make test-internal-mtls
|
||||
make test-login-limit BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-openbao-acl PYTHON=.venv/bin/python
|
||||
make test-worker-isolation
|
||||
make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token
|
||||
make test-leaf-rotation PYTHON=.venv/bin/python
|
||||
make test-telemetry-redaction PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
|
||||
make test-argon-budget
|
||||
.venv/bin/pip-audit -r requirements.txt
|
||||
```
|
||||
|
||||
`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite.
|
||||
@@ -15,10 +24,9 @@ python3 tests/smoke.py --base-url http://localhost:8080
|
||||
|
||||
## Проверка сохранности, lab2+
|
||||
|
||||
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
|
||||
`scripts/restart.sh` перезапускает API и используемые stores, сохраняя volumes и тестовые данные.
|
||||
|
||||
```sh
|
||||
chmod +x scripts/restart.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
```
|
||||
|
||||
@@ -26,10 +34,9 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
|
||||
## Проверка отказа, lab3+
|
||||
|
||||
Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо.
|
||||
`scripts/stop-one.sh` сопоставляет `TARGET_INSTANCE_ID` с hostname API container и останавливает только его. Скрипт оставляет реплику остановленной; восстановление выполняйте отдельно.
|
||||
|
||||
```sh
|
||||
chmod +x scripts/stop-one.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
```
|
||||
|
||||
@@ -50,8 +57,14 @@ make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
|
||||
make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
|
||||
```
|
||||
|
||||
Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях.
|
||||
Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. `make test-internal-mtls` проверяет внутренний API listener из контейнера API: без клиентского сертификата, с недоверенной CA и с недопустимой URI SAN соединение отвергается, а LB identity проходит. `make test-login-limit` проверяет ответ 429 и Retry-After после серии неуспешных входов. Клиентский сертификат внешнему REST-клиенту не требуется: mTLS находится на связи LB→API.
|
||||
|
||||
## Что ещё остаётся доказать
|
||||
`make test-openbao-acl` проверяет все сервисные AppRole: каждая identity читает собственный KV path и получает 403 на paths остальных identities; также проверяется ограниченный TTL токена. `make test-argon-budget` измеряет Argon2id verify latency, burst capacity и число одновременно занятых hash slots внутри API container с его CPU/RAM limits. Эти результаты характеризуют password hashing guard, а не end-to-end capacity стенда. `pip-audit -r requirements.txt` сканирует закреплённые Python runtime-зависимости по advisory database.
|
||||
|
||||
Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
|
||||
`make test-worker-isolation` запускается внутри worker container: проверяет ограниченную PostgreSQL-роль, запрет чтения/перечисления защищённых S3 buckets, разрешённые staging/originals/variants операции и запрет публикации worker в RabbitMQ. Временные probe objects удаляются после теста.
|
||||
|
||||
`make test-worker-rotation ROOT_TOKEN_FILE=...` требует запущенный и распечатанный OpenBao и локальный путь к operator root-token file вне Git. Скрипт создаёт attachment при остановленном worker, выпускает новый SecretID, отзывает прежний, доказывает отказ старого credential, пересоздаёт только worker и проверяет `queued → ready`, checksum original и message link. Root token и SecretID не печатаются. Запуск создаёт синтетические записи и заменяет локальный worker SecretID.
|
||||
|
||||
`make test-leaf-rotation` выпускает и устанавливает новый LB client leaf с прежней URI SAN, пересоздаёт API и HAProxy без build, проверяет отказ старого сертификата, приём нового и неизменность Docker image IDs. `make test-telemetry-redaction` генерирует синтетические credentials и сообщение, затем сканирует API log entries в Loki и трассы Tempo на отсутствие этих значений.
|
||||
|
||||
Smoke и security-проверки рассчитаны на localhost Compose стенд, не на production. Нагрузочный benchmark, persistence hook и failover API выполняются отдельными командами выше; их машинные результаты записываются в `evidence/`.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import resource
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from src import security
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
return ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Measure Argon2id latency and bounded concurrency.")
|
||||
parser.add_argument("--samples", type=int, default=20)
|
||||
parser.add_argument("--concurrency", type=int, default=12)
|
||||
args = parser.parse_args()
|
||||
if args.samples < 1 or args.concurrency < 1:
|
||||
parser.error("samples and concurrency must be positive")
|
||||
|
||||
encoded = security.hash_password("argon2-budget-synthetic-password")
|
||||
durations = []
|
||||
for _ in range(args.samples):
|
||||
started = time.perf_counter()
|
||||
if not security.verify_password("argon2-budget-synthetic-password", encoded):
|
||||
raise RuntimeError("Argon2id self-verification failed")
|
||||
durations.append(time.perf_counter() - started)
|
||||
|
||||
active = 0
|
||||
peak_active = 0
|
||||
lock = threading.Lock()
|
||||
original_slots = security._PASSWORD_HASH_SLOTS
|
||||
|
||||
class TrackingSlots:
|
||||
def acquire(self, timeout=None):
|
||||
nonlocal active, peak_active
|
||||
acquired = original_slots.acquire(timeout=timeout)
|
||||
if acquired:
|
||||
with lock:
|
||||
active += 1
|
||||
peak_active = max(peak_active, active)
|
||||
return acquired
|
||||
|
||||
def release(self):
|
||||
nonlocal active
|
||||
with lock:
|
||||
active -= 1
|
||||
original_slots.release()
|
||||
|
||||
security._PASSWORD_HASH_SLOTS = TrackingSlots()
|
||||
|
||||
def verify_once(_):
|
||||
try:
|
||||
return security.verify_password("argon2-budget-synthetic-password", encoded)
|
||||
except security.PasswordHashCapacityError:
|
||||
return False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
burst_results = list(executor.map(verify_once, range(args.concurrency)))
|
||||
|
||||
print(json.dumps({
|
||||
"argon2_memory_kib": security._ARGON2_MEMORY_KIB,
|
||||
"argon2_iterations": security._ARGON2_ITERATIONS,
|
||||
"argon2_parallelism": security._ARGON2_PARALLELISM,
|
||||
"hash_slots_per_process": security._PASSWORD_HASH_CONCURRENCY,
|
||||
"hash_wait_seconds": security._PASSWORD_HASH_WAIT_SECONDS,
|
||||
"sequential_samples": args.samples,
|
||||
"verify_latency_ms": {
|
||||
"p50": round(statistics.median(durations) * 1000, 3),
|
||||
"p95": round(percentile(durations, 0.95) * 1000, 3),
|
||||
"max": round(max(durations) * 1000, 3),
|
||||
},
|
||||
"burst_requests": args.concurrency,
|
||||
"burst_successes": sum(burst_results),
|
||||
"burst_capacity_rejections": args.concurrency - sum(burst_results),
|
||||
"peak_active_hashes": peak_active,
|
||||
"max_rss_platform_units": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
||||
"platform": sys.platform,
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify API listener client-certificate requirements from inside an API container."""
|
||||
import ssl
|
||||
import http.client
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
HOST = "https://127.0.0.1:8080/health/ready"
|
||||
CA = "/run/secrets/ca_crt"
|
||||
|
||||
|
||||
def context(client_cert=None, client_key=None):
|
||||
result = ssl.create_default_context(cafile=CA)
|
||||
if client_cert:
|
||||
result.load_cert_chain(client_cert, client_key)
|
||||
return result
|
||||
|
||||
|
||||
def rejected(client_cert=None, client_key=None):
|
||||
try:
|
||||
with urllib.request.urlopen(HOST, context=context(client_cert, client_key), timeout=5) as response:
|
||||
response.read(1024)
|
||||
except (urllib.error.URLError, ssl.SSLError, http.client.HTTPException, OSError):
|
||||
return
|
||||
raise AssertionError("API accepted a request with an unauthorized client identity")
|
||||
|
||||
|
||||
def main():
|
||||
rejected()
|
||||
rejected("/run/secrets/wrong_client_cert", "/run/secrets/wrong_client_key")
|
||||
rejected("/run/secrets/foreign_client_cert", "/run/secrets/foreign_client_key")
|
||||
|
||||
with urllib.request.urlopen(
|
||||
HOST,
|
||||
context=context("/run/secrets/lb_tls_cert", "/run/secrets/lb_tls_key"),
|
||||
timeout=5,
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise AssertionError(f"authorized LB identity returned HTTP {response.status}")
|
||||
print("PASS: missing, foreign-CA and non-LB identities rejected; LB identity accepted.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rotate the LB client leaf, pin the replacement, and prove the prior leaf is rejected."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI_DIR = ROOT / "secrets" / "pki"
|
||||
CA_CERT = PKI_DIR / "ca.crt"
|
||||
CA_KEY = PKI_DIR / "ca.key"
|
||||
CERT_PATH = PKI_DIR / "lb-client.crt"
|
||||
KEY_PATH = PKI_DIR / "lb-client.key"
|
||||
BUNDLE_PATH = PKI_DIR / "lb-client.pem"
|
||||
API_PROBE = """
|
||||
import json, ssl, sys, tempfile, urllib.request, urllib.error
|
||||
value = json.load(sys.stdin)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
cert = directory + "/client.crt"
|
||||
key = directory + "/client.key"
|
||||
open(cert, "w").write(value["cert"])
|
||||
open(key, "w").write(value["key"])
|
||||
context = ssl.create_default_context(cafile="/run/secrets/ca_crt")
|
||||
context.load_cert_chain(cert, key)
|
||||
try:
|
||||
with urllib.request.urlopen("https://127.0.0.1:8080/health/ready", context=context, timeout=5) as response:
|
||||
result = {"accepted": response.status == 200}
|
||||
except (urllib.error.URLError, ssl.SSLError, OSError):
|
||||
result = {"accepted": False}
|
||||
print(json.dumps(result))
|
||||
"""
|
||||
|
||||
|
||||
def run(command, *, input_text=None):
|
||||
return subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
input=input_text,
|
||||
text=True,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
|
||||
def compose(*args, input_text=None):
|
||||
return run(["docker", "compose", *args], input_text=input_text)
|
||||
|
||||
|
||||
def write_atomic(path: Path, content: bytes, mode: int) -> None:
|
||||
temporary = path.with_suffix(path.suffix + ".rotation-tmp")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
||||
with os.fdopen(descriptor, "wb") as output:
|
||||
output.write(content)
|
||||
os.chmod(temporary, mode)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def probe(cert: bytes, key: bytes) -> bool:
|
||||
payload = json.dumps(
|
||||
{
|
||||
"cert": cert.decode("ascii"),
|
||||
"key": key.decode("ascii"),
|
||||
}
|
||||
)
|
||||
result = compose("exec", "-T", "api", "python3", "-c", API_PROBE, input_text=payload)
|
||||
return json.loads(result.stdout)["accepted"]
|
||||
|
||||
|
||||
def image_ids() -> set:
|
||||
container_ids = compose("ps", "-q", "api").stdout.split()
|
||||
if len(container_ids) != 2:
|
||||
raise RuntimeError("expected exactly two running API replicas")
|
||||
return {
|
||||
run(["docker", "inspect", "--format", "{{.Image}}", container_id]).stdout.strip()
|
||||
for container_id in container_ids
|
||||
}
|
||||
|
||||
|
||||
def wait_healthy(timeout: int = 90) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
containers = compose("ps", "-q", "api").stdout.split()
|
||||
healthy = len(containers) == 2
|
||||
for container_id in containers:
|
||||
state = json.loads(
|
||||
run(["docker", "inspect", "--format", "{{json .State.Health}}", container_id]).stdout
|
||||
)
|
||||
healthy = healthy and state.get("Status") == "healthy"
|
||||
lb = compose("ps", "-q", "load_balancer").stdout.split()
|
||||
healthy = healthy and len(lb) == 1
|
||||
if healthy:
|
||||
state = json.loads(
|
||||
run(["docker", "inspect", "--format", "{{json .State.Health}}", lb[0]]).stdout
|
||||
)
|
||||
healthy = state.get("Status") == "healthy"
|
||||
if healthy:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError("API and load balancer did not become healthy after certificate rotation")
|
||||
|
||||
|
||||
def create_replacement(directory: Path) -> tuple[bytes, bytes, bytes]:
|
||||
request_path = directory / "lb-client.csr"
|
||||
cert_path = directory / "lb-client.crt"
|
||||
key_path = directory / "lb-client.key"
|
||||
extensions_path = directory / "lb-client.ext"
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl", "req", "-new", "-newkey", "rsa:2048", "-nodes",
|
||||
"-keyout", str(key_path), "-out", str(request_path),
|
||||
"-subj", "/CN=lab7-load-balancer",
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
extensions_path.write_text(
|
||||
"basicConstraints=critical,CA:FALSE\n"
|
||||
"keyUsage=critical,digitalSignature,keyEncipherment\n"
|
||||
"extendedKeyUsage=clientAuth\n"
|
||||
"subjectAltName=URI:spiffe://mtusi2026lab7/load-balancer\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl", "x509", "-req", "-in", str(request_path),
|
||||
"-CA", str(CA_CERT), "-CAkey", str(CA_KEY), "-CAcreateserial",
|
||||
"-out", str(cert_path), "-days", "30", "-sha256",
|
||||
"-extfile", str(extensions_path),
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
cert = cert_path.read_bytes()
|
||||
key = key_path.read_bytes()
|
||||
return cert, key, cert + key
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not CA_CERT.is_file() or not CA_KEY.is_file():
|
||||
raise RuntimeError("local CA certificate and operator-held private key are required")
|
||||
old_cert = CERT_PATH.read_bytes()
|
||||
old_key = KEY_PATH.read_bytes()
|
||||
old_bundle = BUNDLE_PATH.read_bytes()
|
||||
before_images = image_ids()
|
||||
if not probe(old_cert, old_key):
|
||||
raise RuntimeError("current LB certificate was not accepted before rotation")
|
||||
|
||||
replaced = False
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="lab7-lb-leaf-") as directory_name:
|
||||
new_cert, new_key, new_bundle = create_replacement(Path(directory_name))
|
||||
new_fingerprint = hashlib.sha256(ssl.PEM_cert_to_DER_cert(new_cert.decode("ascii"))).hexdigest()
|
||||
write_atomic(CERT_PATH, new_cert, 0o644)
|
||||
write_atomic(KEY_PATH, new_key, 0o600)
|
||||
write_atomic(BUNDLE_PATH, new_bundle, 0o600)
|
||||
replaced = True
|
||||
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
|
||||
wait_healthy()
|
||||
if probe(old_cert, old_key):
|
||||
raise RuntimeError("previous LB certificate remains authorized after rotation")
|
||||
if not probe(new_cert, new_key):
|
||||
raise RuntimeError("replacement LB certificate was not accepted")
|
||||
|
||||
context = ssl.create_default_context(cafile=str(CA_CERT))
|
||||
with urllib.request.urlopen(
|
||||
"https://localhost:8443/health/ready",
|
||||
context=context,
|
||||
timeout=10,
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError("HTTPS ingress did not recover after leaf rotation")
|
||||
|
||||
after_images = image_ids()
|
||||
if before_images != after_images:
|
||||
raise RuntimeError("container image changed; expected rotation without rebuild")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"old_leaf_rejected": True,
|
||||
"new_leaf_accepted": True,
|
||||
"https_ingress_healthy": True,
|
||||
"images_unchanged": True,
|
||||
"new_leaf_sha256": new_fingerprint,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
if replaced:
|
||||
write_atomic(CERT_PATH, old_cert, 0o644)
|
||||
write_atomic(KEY_PATH, old_key, 0o600)
|
||||
write_atomic(BUNDLE_PATH, old_bundle, 0o600)
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import uuid
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Verify the shared login rate limit")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
username = "limit_" + uuid.uuid4().hex[:16]
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body={"username": username, "display_name": "Rate limit test", "password": "Correct-Password-123"},
|
||||
schema="User",
|
||||
)
|
||||
invalid = base64.b64encode((username + ":Wrong-Password-123").encode()).decode("ascii")
|
||||
for _ in range(5):
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
401,
|
||||
headers={"Authorization": "Basic " + invalid},
|
||||
)
|
||||
result = client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
429,
|
||||
headers={"Authorization": "Basic " + invalid},
|
||||
)
|
||||
check(result["json"]["error"]["code"] == "rate_limited", "expected rate_limited response")
|
||||
check(int(result["headers"].get("Retry-After", "0")) > 0, "missing positive Retry-After")
|
||||
print("PASS: sixth failed login in the window is rejected with 429 and Retry-After.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SERVICES = ("api", "worker", "publisher", "migrate", "reconciler")
|
||||
|
||||
|
||||
class BaoClient:
|
||||
def __init__(self, address: str, ca_file: str):
|
||||
self.address = address.rstrip("/")
|
||||
self.context = ssl.create_default_context(cafile=ca_file)
|
||||
|
||||
def request(self, method: str, path: str, payload=None, token: str = ""):
|
||||
body = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
request = urllib.request.Request(
|
||||
self.address + "/v1/" + path,
|
||||
data=body,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=self.context, timeout=5) as response:
|
||||
return response.status, json.loads(response.read(1024 * 1024))
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, {}
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error:
|
||||
raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Check service AppRole path isolation.")
|
||||
parser.add_argument("--address", default="https://localhost:8200")
|
||||
parser.add_argument("--ca-file", required=True)
|
||||
parser.add_argument("--credential-dir", default="secrets/openbao")
|
||||
args = parser.parse_args()
|
||||
credential_dir = Path(args.credential_dir)
|
||||
client = BaoClient(args.address, args.ca_file)
|
||||
|
||||
for service in SERVICES:
|
||||
role_id = (credential_dir / f"{service}_role_id").read_text(encoding="utf-8").strip()
|
||||
secret_id = (credential_dir / f"{service}_secret_id").read_text(encoding="utf-8").strip()
|
||||
status, response = client.request(
|
||||
"POST",
|
||||
"auth/approle/login",
|
||||
{"role_id": role_id, "secret_id": secret_id},
|
||||
)
|
||||
token = response.get("auth", {}).get("client_token")
|
||||
if status != 200 or not token:
|
||||
raise RuntimeError(f"{service} AppRole authentication failed with HTTP {status}")
|
||||
lease_duration = response.get("auth", {}).get("lease_duration")
|
||||
if not isinstance(lease_duration, int) or not 1 <= lease_duration <= 3600:
|
||||
raise RuntimeError(f"{service} AppRole returned an invalid token TTL")
|
||||
own_status, _ = client.request(
|
||||
"GET",
|
||||
f"secret/data/messenger/{service}",
|
||||
token=token,
|
||||
)
|
||||
if own_status != 200:
|
||||
raise RuntimeError(f"{service} cannot read its own scoped KV path (HTTP {own_status})")
|
||||
for other in SERVICES:
|
||||
if other == service:
|
||||
continue
|
||||
denied_status, _ = client.request(
|
||||
"GET",
|
||||
f"secret/data/messenger/{other}",
|
||||
token=token,
|
||||
)
|
||||
if denied_status != 403:
|
||||
raise RuntimeError(
|
||||
f"{service} AppRole accessed {other} KV path (HTTP {denied_status})"
|
||||
)
|
||||
lookup_status, lookup_response = client.request(
|
||||
"GET",
|
||||
"auth/token/lookup-self",
|
||||
token=token,
|
||||
)
|
||||
token_data = lookup_response.get("data", {})
|
||||
if lookup_status != 200 or not 1 <= token_data.get("ttl", 0) <= 3600:
|
||||
raise RuntimeError(
|
||||
f"{service} AppRole token TTL is not bounded (HTTP {lookup_status})"
|
||||
)
|
||||
|
||||
print("PASS: each AppRole reads its own KV path and is denied all other service paths.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -36,7 +36,7 @@ def main():
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=2)
|
||||
c = Client(args.base_url,args.ca_file,timeout=15)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify synthetic credentials and message bodies are absent from Loki logs and Tempo spans."""
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TELEMETRY_PROBE = r"""
|
||||
import json, ssl, sys, time, urllib.parse, urllib.request
|
||||
probe = json.load(sys.stdin)
|
||||
context = ssl.create_default_context(cafile="/run/secrets/ca_crt")
|
||||
def request(url):
|
||||
with urllib.request.urlopen(url, context=context, timeout=8) as response:
|
||||
return json.load(response)
|
||||
log_query = '{service_name="messenger-api"}'
|
||||
deadline = time.monotonic() + 30
|
||||
trace_ids = []
|
||||
log_entries = []
|
||||
while time.monotonic() < deadline:
|
||||
params = urllib.parse.urlencode({"query": log_query, "start": probe["start_ns"], "end": str(time.time_ns()), "limit": "1000"})
|
||||
logs = request("https://loki:3100/loki/api/v1/query_range?" + params)
|
||||
log_entries = [
|
||||
{"labels": stream.get("stream", {}), "line": line}
|
||||
for stream in logs.get("data", {}).get("result", [])
|
||||
for _, line in stream.get("values", [])
|
||||
]
|
||||
search = urllib.parse.urlencode({"limit": "1000", "start": probe["start"], "end": str(int(time.time())), "tags": "service.name=messenger-api"})
|
||||
found = request("https://tempo:3200/api/search?" + search)
|
||||
trace_ids = [item["traceID"] for item in found.get("traces", []) if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"]
|
||||
if trace_ids and log_entries:
|
||||
break
|
||||
time.sleep(1)
|
||||
if not trace_ids:
|
||||
raise RuntimeError("no API message trace appeared in Tempo within the deadline")
|
||||
payloads = [json.dumps(log_entries, ensure_ascii=False)]
|
||||
for trace_id in trace_ids:
|
||||
payloads.append(json.dumps(request("https://tempo:3200/api/traces/" + trace_id), ensure_ascii=False))
|
||||
leaked = [value for value in probe["values"] if any(value in payload for payload in payloads)]
|
||||
print(json.dumps({"trace_count": len(trace_ids), "log_entries": len(log_entries), "leak_count": len(leaked)}))
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = arguments("Verify request secrets and message bodies are excluded from telemetry")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
canary = "telemetry-canary-" + uuid.uuid4().hex
|
||||
username = "u_probe_" + uuid.uuid4().hex[:12]
|
||||
password = "Password-" + uuid.uuid4().hex
|
||||
start_time = time.time()
|
||||
user = client.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body={"username": username, "display_name": canary, "password": password},
|
||||
schema="User",
|
||||
)["json"]
|
||||
auth_value = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||||
session = client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "Basic " + auth_value},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
check(session["user"]["id"] == user["id"], "telemetry probe user session did not authenticate")
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/users/me",
|
||||
401,
|
||||
headers={"Authorization": "Bearer " + canary},
|
||||
)
|
||||
client.message(alice, chat, canary)
|
||||
values = [canary, username, password, auth_value, session["token"], alice["token"]]
|
||||
probe = {
|
||||
"values": values,
|
||||
"start": str(max(0, int(start_time) - 5)),
|
||||
"start_ns": str(max(0, int(start_time - 5) * 1_000_000_000)),
|
||||
}
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker", "compose", "exec", "-T", "api",
|
||||
"python3", "-c", TELEMETRY_PROBE,
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=json.dumps(probe),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
telemetry = json.loads(result.stdout)
|
||||
check(telemetry["trace_count"] > 0, "no API traces were scanned")
|
||||
check(telemetry["leak_count"] == 0, "synthetic credentials or message content appeared in telemetry")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"synthetic_values_checked": len(values),
|
||||
"api_traces_scanned": telemetry["trace_count"],
|
||||
"log_entries_scanned": telemetry["log_entries"],
|
||||
"leaks": 0,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,54 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from src.security import (
|
||||
PasswordHashCapacityError,
|
||||
allowed_client_identity,
|
||||
hash_password,
|
||||
password_hash_needs_upgrade,
|
||||
verify_password,
|
||||
)
|
||||
|
||||
|
||||
class PasswordHashTests(unittest.TestCase):
|
||||
def test_argon2id_verifies_unicode_password(self):
|
||||
encoded = hash_password("корректный-password-123")
|
||||
self.assertTrue(encoded.startswith("$argon2id$"))
|
||||
self.assertIn("m=19456,t=2,p=1", encoded)
|
||||
self.assertTrue(verify_password("корректный-password-123", encoded))
|
||||
self.assertFalse(verify_password("другой-password-123", encoded))
|
||||
self.assertFalse(password_hash_needs_upgrade(encoded))
|
||||
oversized = "$argon2id$v=19$m=999999,t=2,p=1$c2FsdA$ZGlnaWVzdA"
|
||||
self.assertFalse(verify_password("attacker-password-123", oversized))
|
||||
|
||||
def test_legacy_scrypt_hash_is_verified_and_marked_for_upgrade(self):
|
||||
password = "legacy-password-123"
|
||||
salt = b"lab6-test-salt!!"
|
||||
digest = hashlib.scrypt(
|
||||
password.encode(), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
|
||||
)
|
||||
stored = "scrypt$32768$8$1${}${}".format(
|
||||
base64.urlsafe_b64encode(salt).decode(),
|
||||
base64.urlsafe_b64encode(digest).decode(),
|
||||
)
|
||||
self.assertTrue(verify_password(password, stored))
|
||||
self.assertTrue(password_hash_needs_upgrade(stored))
|
||||
|
||||
def test_only_lb_service_identity_is_authorized(self):
|
||||
trusted = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/load-balancer")]}
|
||||
other = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/other-service")]}
|
||||
self.assertTrue(allowed_client_identity(trusted))
|
||||
self.assertFalse(allowed_client_identity(other))
|
||||
self.assertFalse(allowed_client_identity({}))
|
||||
|
||||
def test_password_hash_capacity_fails_fast(self):
|
||||
with patch("src.security._PASSWORD_HASH_SLOTS") as slots:
|
||||
slots.acquire.return_value = False
|
||||
with self.assertRaises(PasswordHashCapacityError):
|
||||
verify_password("password", "$argon2id$malformed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Revoke the worker AppRole SecretID and recover a queued attachment with a replacement."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets" / "openbao"
|
||||
|
||||
|
||||
class BaoClient:
|
||||
def __init__(self, ca_file: str, root_token_file: str):
|
||||
self.address = "https://localhost:8200"
|
||||
self.context = ssl.create_default_context(cafile=ca_file)
|
||||
self.root_token = Path(root_token_file).read_text(encoding="utf-8").strip()
|
||||
check(bool(self.root_token), "operator root-token file is empty")
|
||||
|
||||
def request(self, method: str, path: str, payload=None, token=None):
|
||||
body = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token is not None:
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
elif self.root_token:
|
||||
headers["X-Vault-Token"] = self.root_token
|
||||
request = urllib.request.Request(
|
||||
self.address + "/v1/" + path,
|
||||
data=body,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=self.context, timeout=8) as response:
|
||||
data = response.read(1024 * 1024)
|
||||
return response.status, json.loads(data) if data else {}
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, {}
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error:
|
||||
raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None
|
||||
|
||||
|
||||
def compose(*args) -> None:
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def save_secret_id(value: str) -> None:
|
||||
path = SECRET_DIR / "worker_secret_id"
|
||||
temporary = path.with_suffix(".tmp")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def rotate_worker_secret(bao: BaoClient) -> None:
|
||||
role_id = (SECRET_DIR / "worker_role_id").read_text(encoding="utf-8").strip()
|
||||
old_secret_id = (SECRET_DIR / "worker_secret_id").read_text(encoding="utf-8").strip()
|
||||
check(bool(role_id) and bool(old_secret_id), "worker AppRole credential files are empty")
|
||||
|
||||
status, created = bao.request(
|
||||
"POST",
|
||||
"auth/approle/role/worker/secret-id",
|
||||
{},
|
||||
)
|
||||
new_secret_id = created.get("data", {}).get("secret_id")
|
||||
check(status == 200 and isinstance(new_secret_id, str), "OpenBao did not issue a replacement SecretID")
|
||||
save_secret_id(new_secret_id)
|
||||
|
||||
status, login = bao.request(
|
||||
"POST",
|
||||
"auth/approle/login",
|
||||
{"role_id": role_id, "secret_id": new_secret_id},
|
||||
token="",
|
||||
)
|
||||
token = login.get("auth", {}).get("client_token")
|
||||
check(status == 200 and isinstance(token, str), "replacement worker SecretID did not authenticate")
|
||||
status, _ = bao.request(
|
||||
"GET",
|
||||
"secret/data/messenger/worker",
|
||||
token=token,
|
||||
)
|
||||
check(status == 200, "replacement worker identity cannot read its own secret path")
|
||||
|
||||
status, _ = bao.request(
|
||||
"POST",
|
||||
"auth/approle/role/worker/secret-id/destroy",
|
||||
{"secret_id": old_secret_id},
|
||||
)
|
||||
check(status == 204, "OpenBao did not revoke the previous worker SecretID")
|
||||
status, _ = bao.request(
|
||||
"POST",
|
||||
"auth/approle/login",
|
||||
{"role_id": role_id, "secret_id": old_secret_id},
|
||||
token="",
|
||||
)
|
||||
check(status == 400, "revoked worker SecretID still authenticates")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
def configure(parser):
|
||||
parser.add_argument("--root-token-file", required=True)
|
||||
|
||||
args = arguments("Verify queued attachment recovery after worker SecretID rotation", configure)
|
||||
ca_file = args.ca_file or str(ROOT / "secrets" / "pki" / "ca.crt")
|
||||
client = Client(args.base_url, ca_file)
|
||||
bao = BaoClient(ca_file, args.root_token_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
worker_stopped = False
|
||||
try:
|
||||
compose("stop", "worker")
|
||||
worker_stopped = True
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not queued while the worker was stopped")
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to its message")
|
||||
|
||||
rotate_worker_secret(bao)
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
worker_stopped = False
|
||||
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
original_bytes = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(original_bytes == original, "rotated worker changed the original attachment bytes")
|
||||
page = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
recovered = next(item for item in page["items"] if item["id"] == message["id"])
|
||||
check(recovered["attachment_ids"] == [attachment["id"]], "message lost its attachment link")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"old_worker_secret_id_revoked": True,
|
||||
"replacement_worker_secret_id_authenticated": True,
|
||||
"attachment_state": [attachment["status"], ready["status"]],
|
||||
"original_sha256_preserved": True,
|
||||
"message_attachment_link_preserved": True,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if worker_stopped:
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify the worker identity is restricted to its required database, S3, and broker actions."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from src.jobs import connect
|
||||
from src.object_store import ObjectStore
|
||||
from src.store import database_url
|
||||
|
||||
|
||||
def check_database() -> dict:
|
||||
with psycopg.connect(database_url()) as connection:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT current_user")
|
||||
role = cursor.fetchone()[0]
|
||||
cursor.execute("SELECT id, status FROM attachments LIMIT 1")
|
||||
cursor.fetchall()
|
||||
try:
|
||||
cursor.execute("SELECT id FROM users LIMIT 1")
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
connection.rollback()
|
||||
users_denied = True
|
||||
else:
|
||||
raise AssertionError("worker can read users")
|
||||
if role != "messenger_worker":
|
||||
raise AssertionError(f"unexpected worker database role: {role}")
|
||||
return {"role": role, "attachments_read": True, "users_read_denied": users_denied}
|
||||
|
||||
|
||||
def check_s3() -> dict:
|
||||
client = ObjectStore().client
|
||||
results = {}
|
||||
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants"):
|
||||
try:
|
||||
client.list_objects_v2(Bucket=bucket, MaxKeys=1)
|
||||
except ClientError as error:
|
||||
code = error.response.get("Error", {}).get("Code")
|
||||
if code not in ("AccessDenied", "403"):
|
||||
raise
|
||||
results[f"{bucket}_list_denied"] = True
|
||||
else:
|
||||
raise AssertionError(f"worker can list {bucket}")
|
||||
|
||||
for bucket, key in (
|
||||
("messenger-attachments", f"original/{uuid.uuid4()}"),
|
||||
("messenger-variants", f"variants/{uuid.uuid4()}"),
|
||||
):
|
||||
try:
|
||||
client.get_object(Bucket=bucket, Key=key)
|
||||
except ClientError as error:
|
||||
code = error.response.get("Error", {}).get("Code")
|
||||
if code not in ("AccessDenied", "403"):
|
||||
raise
|
||||
results[f"{bucket}_read_denied"] = True
|
||||
else:
|
||||
raise AssertionError(f"worker can read objects from {bucket}")
|
||||
|
||||
probe = str(uuid.uuid4())
|
||||
allowed_objects = (
|
||||
("messenger-staging", f"staging/isolation-probe/{probe}", True),
|
||||
("messenger-attachments", f"original/isolation-probe/{probe}", False),
|
||||
("messenger-variants", f"variants/isolation-probe/{probe}", False),
|
||||
)
|
||||
for bucket, key, verify_read in allowed_objects:
|
||||
try:
|
||||
client.put_object(Bucket=bucket, Key=key, Body=b"worker-scope-probe")
|
||||
if verify_read:
|
||||
response = client.get_object(Bucket=bucket, Key=key)
|
||||
try:
|
||||
if response["Body"].read() != b"worker-scope-probe":
|
||||
raise AssertionError("worker staging read returned unexpected data")
|
||||
finally:
|
||||
response["Body"].close()
|
||||
results[f"{bucket}_write"] = True
|
||||
finally:
|
||||
try:
|
||||
client.delete_object(Bucket=bucket, Key=key)
|
||||
except ClientError as error:
|
||||
if error.response.get("Error", {}).get("Code") not in ("NoSuchKey", "404"):
|
||||
raise
|
||||
results["messenger-staging_read"] = True
|
||||
results["temporary_objects_removed"] = True
|
||||
return results
|
||||
|
||||
|
||||
def check_broker() -> dict:
|
||||
connection = connect()
|
||||
channel = connection.channel()
|
||||
try:
|
||||
try:
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key="attachments.process.v1",
|
||||
body=b"worker-isolation-probe",
|
||||
)
|
||||
channel.close()
|
||||
except pika.exceptions.ChannelClosedByBroker as error:
|
||||
if error.reply_code != 403:
|
||||
raise
|
||||
publish_denied = True
|
||||
else:
|
||||
publish_denied = False
|
||||
if not publish_denied:
|
||||
raise AssertionError("worker can publish to the processing queue")
|
||||
return {"broker_identity_connected": True, "publish_denied": publish_denied}
|
||||
finally:
|
||||
if channel.is_open:
|
||||
channel.close()
|
||||
if connection.is_open:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result = {
|
||||
"database": check_database(),
|
||||
"s3": check_s3(),
|
||||
"broker": check_broker(),
|
||||
}
|
||||
print(json.dumps({"result": "PASS", **result}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user