выполение на 5
Skeleton contracts / contracts (pull_request) Canceled after 0s

This commit is contained in:
kaktuz20000
2026-11-04 19:22:35 +03:00
parent 20afc2c790
commit 5a6fcff3ef
90 changed files with 8848 additions and 130 deletions
+14 -37
View File
@@ -1,41 +1,18 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
# Local Compose settings only. Credentials and private keys live under ignored secrets/.
APP_PORT=8443
API_REPLICAS=2
API_CPU_LIMIT=1.0
API_MEMORY_LIMIT=512M
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=https://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
MAX_CONCURRENT_UPLOADS=10
WORKER_CONCURRENCY=1
WORKER_REPLICAS=1
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
PUBLIC_BASE_URL=https://localhost:8443
TLS_CA_FILE=/run/secrets/ca.crt
TLS_CERT_FILE=/run/secrets/service.crt
TLS_KEY_FILE=/run/secrets/service.key
BAO_ADDR=https://openbao:8200
BAO_CACERT=/run/secrets/bao-ca.crt
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
ARGON2_MEMORY_KIB=19456
ARGON2_ITERATIONS=2
ARGON2_PARALLELISM=1
RECONCILE_INTERVAL_SECONDS=900
RECONCILE_GRACE_SECONDS=3600
GRAFANA_PORT=3000
OTEL_SDK_DISABLED=false
OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=1.0
+22
View File
@@ -0,0 +1,22 @@
FROM python:3.12.7-slim
WORKDIR /app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY . /app
RUN chown -R app:app /app
USER app
EXPOSE 8080
CMD ["python3", "src/server.py"]
+3 -3
View File
@@ -31,8 +31,8 @@
Исходные README/контракт/публичные tests сохраняются, реализация переносится студентом между лабами через собственные исходники и миграции. Исправление общей спецификации публикуйте отдельным изменением, объясняя влияние на все затронутые лабораторные.
## Проверка этой версии skeleton
## Проверка текущей реализации
На 2026-09-10 выполнены структурные проверки и полная валидация всех семи OpenAPI/JSON Schema, проверка локальных Markdown-ссылок/YAML и декодирование PNG fixture. HTTP-клиенты проверены на временном изолированном тестовом стенде: позитивный smoke 1–7, отрицательные варианты неправильного sender, pagination, ACL, logout, bytes и пустого API; также пути persistence/failover/load и внешнего TLS.
На 2026-11-04 выполнены `make check`, полная OpenAPI-валидация, тесты парольных примитивов, проверка Python 3.9 syntax grammar, `docker compose config --quiet` и сборка API image. Реальный `MutualTLSServer` проверен с локальными сертификатами: запрос без client certificate, с foreign CA и с неразрешённым URI SAN отвергнуты; разрешённая LB identity принята.
Временный стенд не включён в студенческие каталоги и не является эталонным решением. Реальное приложение, Compose, PostgreSQL/S3/broker и mTLS/OpenBao в skeleton отсутствуют: их работоспособность не заявляется. Docker CLI в среде подготовки не был доступен. Это проверка материалов и тестовых драйверов, не приёмка будущих решений студентов.
Полный Compose smoke, OpenBao runtime allow/deny, persistence/failover и нагрузочный прогон следует выполнять на изолированном стенде после operator init/unseal и создания локальных credentials.
+65 -1
View File
@@ -4,7 +4,10 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-security
.PHONY: help check validate test test-primitives test-internal-mtls test-login-limit test-openbao-acl test-worker-isolation test-worker-rotation test-leaf-rotation test-telemetry-redaction test-argon-budget load benchmark-observability \
test-attachments test-idempotency-race test-reconcile test-worker test-pipeline-faults \
test-pipeline-replay test-outbox-recovery test-worker-recovery test-reconcile-integration \
test-upload-concurrency test-persistence test-failover test-security
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -18,9 +21,70 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-primitives:
$(PYTHON) -m unittest discover -s tests -p 'test_*.py'
test-internal-mtls:
docker compose exec -T api python3 /app/tests/internal_mtls.py
test-login-limit:
$(PYTHON) tests/login_rate_limit.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-openbao-acl:
$(PYTHON) tests/openbao_acl.py --ca-file "$(CURDIR)/secrets/pki/ca.crt"
test-worker-isolation:
docker compose exec -T worker python3 /app/tests/worker_isolation.py
test-worker-rotation:
@test -n "$(ROOT_TOKEN_FILE)" || (echo "Set ROOT_TOKEN_FILE to an operator-only root-token file"; exit 2)
$(PYTHON) tests/worker_identity_rotation.py --ca-file "$(CURDIR)/secrets/pki/ca.crt" --root-token-file "$(ROOT_TOKEN_FILE)"
test-leaf-rotation:
$(PYTHON) tests/leaf_certificate_rotation.py
test-telemetry-redaction:
$(PYTHON) tests/telemetry_redaction.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-argon-budget:
docker compose exec -T api python3 /app/tests/argon2_budget.py
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
benchmark-observability:
$(PYTHON) student-tests/track1_benchmark.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency-race:
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile:
$(PYTHON) student-tests/test_reconcile.py
test-worker:
$(PYTHON) student-tests/test_worker.py
test-pipeline-faults:
$(PYTHON) student-tests/test_pipeline_faults.py
test-pipeline-replay:
$(PYTHON) student-tests/test_pipeline_replay_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-outbox-recovery:
$(PYTHON) student-tests/test_outbox_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-worker-recovery:
$(PYTHON) student-tests/test_worker_recovery_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-reconcile-integration:
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-upload-concurrency:
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+25 -19
View File
@@ -2,15 +2,13 @@
Защитить работающий мессенджер на границах доверия и показать защиту отрицательными проверками: чужой доступ, неверный сертификат, недостаточные права и отзыв credentials.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
Основа приложения и проверок перенесена из lab6. В этой работе сохранены REST API, PostgreSQL, S3-вложения, RabbitMQ worker и OpenTelemetry-стенд; внешний ingress работает по HTTPS, связь LB→API — по mTLS с проверкой URI SAN, пароли хешируются Argon2id, а рабочие secrets выдаются сервисам через OpenBao AppRole.
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
## Откуда начинаем
## Состав стенда
Перенесите минимум lab6. Бизнес-API сохраняется, внешний endpoint теперь HTTPS. Новые секреты и ключи создаются для стенда, не поставляются в skeleton.
**Архитектура:** Клиент — TLS → LB — mTLS → API; API/worker — TLS (+ клиентские cert где требуется матрицей) → зависимости; OpenBao выдаёт scoped credentials. Grafana/OTel остаются внутри доверенной сети.
**Архитектура:** клиент — TLS → HAProxy — mTLS → API; API, worker, publisher, migrator и reconciler получают секреты через отдельные AppRole policies. Основные сервисы и observability сохраняют архитектуру lab6.
## Что сделать
@@ -50,33 +48,41 @@
## Как начать и проверить
```sh
# Работает прямо в skeleton, Python 3.9+; приложение не запускается:
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
cp .env.example .env
.venv/bin/python scripts/bootstrap-secrets.py
.venv/bin/python scripts/bootstrap-pki.py
make check
# После реализации запустите свой сервер/стенд по REPORT.md.
# По умолчанию тест использует https://localhost:8443.
make test
```
# Сначала хранилища и OpenBao:
docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao
# Оператор инициализирует и unseal-ит OpenBao; root token/unseal shares
# сохраняются вне репозитория и передаются только скрипту bootstrap:
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal
# Повторите unseal для требуемого порога shares.
python3 scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token
`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость.
```sh
# Compose-вариант после реализации:
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождитесь /health/ready; затем make test.
# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md.
make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-worker-isolation
make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token
```
Для HTTPS передайте публичный CA: `make test CA_FILE=/absolute/path/to/ca.crt`. CA certificate можно распространять; private keys создаются локально. `make test-security` проверяет только внешний TLS trust; внутренние отрицательные тесты реализуйте по [trust matrix](contracts/trust-matrix.md).
`.env.example` содержит открытые локальные настройки. PKI и локальные bootstrap secrets создаются в игнорируемом Git каталоге `secrets/`. `compose.example.yaml` — worksheet; запускаемый стенд находится в `compose.yaml`. Образы закреплены версиями, данные приложения и OpenBao хранятся в named volumes. Обычная остановка сохраняет учебные данные (`docker compose down`); намеренное удаление данных выполняется отдельно (`docker compose down -v`).
Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически):
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
make test-primitives PYTHON=.venv/bin/python
make test-openbao-acl PYTHON=.venv/bin/python
make test-argon-budget
.venv/bin/pip-audit -r requirements.txt
```
Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте.
+89 -16
View File
@@ -1,37 +1,110 @@
# Отчёт — лабораторная 7
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: в исходном отчёте lab6 не указаны.
- Целевая оценка: 5; выбранный трек: 1 (компрометация worker). Ниже указаны повторяемая методика и результаты на локальном Compose-стенде.
- Commit/PR: не создан.
- Использованные источники, библиотеки и помощь ИИ: собственная реализация lab6; README/COURSE и контракты lab7; документация Argon2, Python `ssl`, OpenBao AppRole/ACL и Docker Compose. Использованы psycopg, boto3, Pillow, pika, OpenTelemetry и argon2-cffi (версии закреплены в `requirements.txt`). ИИ использовался для переноса, адаптации и проверки кода.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Compose v2, OpenSSL и Python 3.11+. Образы зафиксированы в `compose.yaml`; приложение собирается из `python:3.12.7-slim`. На машине разработки использован Python 3.13.2.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
cp .env.example .env
.venv/bin/python scripts/bootstrap-secrets.py
.venv/bin/python scripts/bootstrap-pki.py
docker compose --env-file .env config --quiet
# Поднять зависимости и OpenBao до приложения.
docker compose --env-file .env up -d postgres rabbitmq s3-core s3 openbao
# Оператор выполняет init и unseal; shares и root token хранятся вне репозитория.
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator init
docker compose exec -it -e BAO_CACERT=/run/secrets/ca_crt openbao bao operator unseal
# Повторить unseal до достижения порога, затем:
.venv/bin/python scripts/bootstrap-openbao.py --root-token-file /secure/path/openbao-root-token
docker compose --env-file .env up -d --build
make test PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-security PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-primitives PYTHON=.venv/bin/python
make test-internal-mtls
make test-login-limit PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
```
Для остановки без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных и OpenBao state: `docker compose --env-file .env down -v`. Скрипты создают credentials и ключи только локально в исключённом из Git каталоге `secrets/`; не копируйте его в репозиторий. Секреты и unseal material не помещать в командную строку, отчёт или evidence.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перенесены приложение, миграции, обработка вложений и тестовые сценарии lab6. PostgreSQL хранит пользователей, чаты, членство, сообщения, сессии, idempotency state и metadata вложений. SeaweedFS хранит оригиналы и variants; RabbitMQ обеспечивает at-least-once обработку через transactional outbox, отдельный publisher и worker; reconciler сверяет состояние. OTel Collector направляет сигналы в Prometheus, Loki и Tempo, Grafana загружает provisioning из репозитория.
HAProxy завершает внешний TLS на `localhost:8443`; сертификат ingress содержит SAN для `localhost` и `127.0.0.1`. HAProxy проверяет CA и имя API-сервера и предъявляет клиентский сертификат. API требует клиентский сертификат из локального CA и дополнительно разрешает только URI SAN `spiffe://mtusi2026lab7/load-balancer`. API listener не опубликован на host; internal health-check также использует TLS/mTLS.
Пароли записываются как Argon2id (`m=19456 KiB`, `t=2`, `p=1`); параметры hash проверяются до верификации и ограничены верхним пределом 64 MiB. Одновременно выполняется не более четырёх password hash операций в каждом API процессе; при исчерпании слотов запрос ограниченно ждёт 100 ms и получает 503/Retry-After, а не неограниченно аллоцирует память. Проверка legacy scrypt поддержана для миграции пользователей lab6; при успешном входе старый hash обновляется. Токены сессий продолжают храниться в PostgreSQL как SHA-256 hashes. Ограничение входа — пять попыток на username и шестьдесят на source IP за минуту; HAProxy заменяет `X-Real-IP`, поэтому значение от внешнего клиента не доверяется. Состояние общее для API-реплик, ответ блокировки — 429 с Retry-After.
OpenBao использует TLS, file storage в persistent volume и KV v2. Bootstrap создаёт отдельные AppRole и policy path для API, worker, publisher, migrator и reconciler. Приложения получают только собственные static KV values; операторский root token и unseal keys не передаются контейнерам. SecretID ограничен 24 часами, token TTL — 15 минут (max 1 час). Сертификат CA и private keys создаются скриптом на локальном стенде.
Фактические адреса приложения заданы в `compose.yaml`; `.env.example` — шаблон параметров локального запуска. Из переменных примера Compose использует порт ingress, число/лимиты API, session TTL, worker concurrency и telemetry sampling. Runtime secrets читаются через AppRole; в контейнерах нет root token. Python-сервисы работают как непривилегированный `app`, с `read_only`, tmpfs, `cap_drop: ALL` и `no-new-privileges`. Одноразовый `openbao-data-init` запускается от root только для установки владельца persistent volume и имеет единственную capability `CHOWN`; сам OpenBao drops all, оставляя `SETUID`/`SETGID`, нужные upstream entrypoint.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура и контракт | `make check` | PASS: OpenAPI structure, local refs, JSON и Python syntax | `scripts/check_contract.py` |
| Схемы API | `make validate PYTHON=.venv/bin/python` | PASS: OpenAPI 3.1 и JSON Schema | `scripts/validate_openapi.py` |
| Криптографические примитивы | `make test-primitives PYTHON=.venv/bin/python` | PASS: Argon2id, legacy scrypt verify/upgrade, URI SAN allowlist | `tests/test_security_primitives.py` |
| Compose topology | `docker compose --env-file .env.example config --quiet` после генерации локальных секретов/PKI | PASS: Compose конфигурация разбирается | `compose.yaml` |
| Локальная PKI | `openssl verify -CAfile secrets/pki/ca.crt ...` | PASS: ingress, API и LB client chain; client URI SAN содержит разрешённую identity | `scripts/bootstrap-pki.py` |
| API smoke over HTTPS | `make test BASE_URL=https://localhost:8443 CA_FILE=...` | PASS на работающем Compose стенде | `tests/smoke.py` |
| Внешняя проверка TLS trust | `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: доверенный TLS проходит, пустой trust store отвергается | `tests/security.py` |
| Внутреннее mTLS | `make test-internal-mtls` | PASS: без cert, foreign CA и неверный URI SAN отклонены; разрешённая LB identity принята | `tests/internal_mtls.py`, `src/app.py`, `haproxy.cfg` |
| Login rate limit | `make test-login-limit BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: шестая неуспешная попытка получает 429/Retry-After | `tests/login_rate_limit.py`, `migrations/006_login_rate_limits.sql` |
| Внутренние TLS-соединения | Работающий Compose smoke, S3 attachment processing, worker broker consumer и проверки datasource из Grafana | PASS: PostgreSQL TLS, S3 HTTPS/mTLS, RabbitMQ AMQPS/mTLS, OTLP HTTPS/mTLS; из Grafana проверена CA-валидация Prometheus/Loki/Tempo | `compose.yaml`, `infra/postgres/`, `infra/otel/`, `src/object_store.py`, `src/jobs.py` |
| OpenBao AppRole | `make test-openbao-acl PYTHON=.venv/bin/python` | PASS: пять сервисов читают собственные KV paths, cross-service read получает 403; token TTL ограничен | `tests/openbao_acl.py`, `scripts/bootstrap-openbao.py`, `infra/openbao/` |
| Worker least privilege | `make test-worker-isolation` | PASS: DB identity `messenger_worker`; чтение `users`, S3 List и чтение originals/variants запрещены; нужные staging/originals/variants операции разрешены; RabbitMQ publish отклонён с 403 | `tests/worker_isolation.py`, `evidence/worker-isolation.json` |
| Track 1: компрометация и recovery | `make test-worker-rotation ROOT_TOKEN_FILE=...` | PASS: старая AppRole SecretID отозвана и отвергнута; новая identity аутентифицируется; задача `queued → ready`, SHA-256 original и message link сохранены | `tests/worker_identity_rotation.py`, `evidence/worker-rotation.json` |
| Ротация LB leaf-сертификата | `make test-leaf-rotation` | PASS: старый сертификат отвергнут, новый принят, HTTPS ingress healthy, Docker image IDs не изменились | `tests/leaf_certificate_rotation.py`, `evidence/leaf-rotation.json` |
| Redaction telemetry | `make test-telemetry-redaction BASE_URL=https://localhost:8443 CA_FILE=...` | PASS: 6 синтетических значений отсутствуют в 11 фактических Loki entries и 1 Tempo trace | `tests/telemetry_redaction.py`, `evidence/telemetry-redaction.json` |
| Сохранность при restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: подтверждённое сообщение, attachment content и существующая сессия сохранены | `tests/resilience.py`, `evidence/persistence.json` |
| Отказ одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: две реплики до отказа; 108 успешных записей, 0 ошибок; первое восстановление за 3.463 s | `tests/resilience.py`, `evidence/failover.json` |
| Argon2 resource budget | `make test-argon-budget` на API с лимитом 1 CPU/512 MiB | 20 verify операций: p50 23.007 ms, p95 28.288 ms, max 32.157 ms; из 12 одновременных запросов 6 выполнились и 6 получили capacity rejection, peak 4 активных hash операции, max RSS 162220 KiB (Linux ru_maxrss) | `tests/argon2_budget.py`, `evidence/argon2-budget.json` |
| Python dependency scan | `.venv/bin/pip-audit -r requirements.txt` | PASS: 39 runtime packages, 0 известных уязвимостей в момент сканирования | `requirements.txt`, `evidence/dependency-audit.json` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Трек 1 — последствия компрометации worker
**Гипотеза.** Получив worker identity, атакующий может читать/писать staging bucket, писать originals/variants и потреблять process queue. Он не может читать пользовательские записи, перечислять buckets или читать originals/variants, публиковать свои задачи либо читать чужие OpenBao paths. После отзыва SecretID и его замены ожидающая задача должна завершиться без потери вложения или связи с сообщением.
**Методика.** На работающем Compose стенде выполнены `make test-worker-isolation` и `make test-worker-rotation ROOT_TOKEN_FILE=<operator-only-file>`. Первый сценарий проверяет реальные PostgreSQL grants, SeaweedFS `AccessDenied`, staging read/write, запись originals/variants, очистку временных объектов и broker ACL `403`. Второй останавливает worker, создаёт attachment и message link, выдаёт новую AppRole SecretID, отзывает прежнюю, проверяет отказ старого credential, пересоздаёт только worker и ждёт окончания обработки через публичный API. Root token и SecretID в вывод не включаются.
**Результат и сравнение.** В исходной общей схеме worker наследовал права общих application credentials; после разделения его DB role и S3/RabbitMQ identity имеют отдельные allowlists. В текущем измерении worker прочитал attachments, но получил отказ на `users`; S3 write для originals/variants и staging read/write прошли, а List и чтение originals/variants завершились `AccessDenied`; RabbitMQ отклонил публикацию с 403. Прежняя SecretID не прошла AppRole login, новая identity получила только собственный KV path. Attachment перешёл из `queued` в `ready`; SHA-256 исходных байтов и ссылка на attachment в сообщении сохранились. Машинные результаты сохранены в `evidence/worker-isolation.json` и `evidence/worker-rotation.json`.
**Границы эксперимента.** Проверялись заданные операции одной worker identity и один queued recovery сценарий на локальном Compose стенде; это не нагрузочный тест и не модель компрометации Docker host.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Проверены сохранность при restart, отказ одной API-реплики и восстановление записи; результаты приведены в таблице и JSON evidence. Отдельно измерялась стоимость Argon2 verify в ограниченном API container. Повторяемый профиль для учебного стенда:
```sh
make load PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 \
CA_FILE="$PWD/secrets/pki/ca.crt"
make test-persistence ACTION_SCRIPT=scripts/restart.sh \
BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-failover ACTION_SCRIPT=scripts/stop-one.sh \
BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
```
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Нет клиентского сертификата / foreign CA / неверный URI SAN | API TLS listener отвергает соединение; LB identity проходит | Runtime тест: все три недоверенных запроса отклонены, LB identity принята | TLS-handshake |
| Недоверенный CA внешнего endpoint | Проверка сертификата завершается TLS verification error | HTTPS smoke и проверка trust store выполнены | TLS-handshake |
| Пять неуспешных login попыток в пределах минуты | Следующая попытка получает 429 и Retry-After | Runtime тест: шестая попытка отклонена с 429/Retry-After | Окно лимита — 60 секунд |
| Отзыв worker SecretID при ожидающей задаче | Старое credential отклоняется, новая identity продолжает обработку | Runtime тест: old login denied; attachment `queued → ready`, содержимое и message link сохранены | Проверен один recovery цикл |
| Restart API и stores | Сохранение подтверждённых данных, attachment content и существующей сессии | Runtime test: все данные доступны после restart | Проверка готовности после возврата сервисов |
| Отказ одной API-реплики | Успешные записи через оставшуюся реплику, без дублей и потери подтверждённых данных | 108 успешных записей, 0 ошибок; 2 реплики до отказа | Первое восстановление за 3.463 s |
## Ограничения и следующие шаги
`tests/load.py` — closed-loop генератор; его успешный HTTP throughput не является точным числом DB commits и не компенсирует coordinated omission. Измерения необходимо выполнять на одном и том же составе стенда с зафиксированными concurrency, длительностью, прогревом, аппаратными ресурсами и размером данных; сохранять вывод в `evidence/` без credentials и токенов.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Дополнительные измерения
Ротация LB leaf-сертификата проверена без пересборки образов: прежний сертификат отклоняется, новый проходит ingress health check. Redaction проверен на фактических записях Loki и трассе Tempo после генерации синтетических credentials и сообщения. Фактические протоколы и identities каждой связи зафиксированы в [матрице доверия](contracts/trust-matrix.md); остальные проверки и команды воспроизведения приведены в [tests/README.md](tests/README.md).
+871
View File
@@ -0,0 +1,871 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
- postgres_tls_cert
- ca_crt
command:
- postgres
- -c
- ssl=on
- -c
- ssl_cert_file=/tls/server.crt
- -c
- ssl_key_file=/tls/server.key
- -c
- ssl_min_protocol_version=TLSv1.2
- -c
- hba_file=/etc/postgresql/pg_hba.conf
volumes:
- postgres_data:/var/lib/postgresql/data
- postgres_tls:/tls
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
depends_on:
postgres-tls-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
rabbitmq:
image: rabbitmq:3.13.7-management-alpine
secrets:
- source: rabbitmq_config
target: /etc/rabbitmq/conf.d/10-defaults.conf
- ca_crt
- rabbitmq_tls_cert
- rabbitmq_tls_key
- rabbitmq_definitions
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
restart: unless-stopped
s3-core:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- server
- -dir=/data
- -filer=true
- -s3=false
- -volume.max=21
volumes:
- seaweed_data:/data
healthcheck:
test:
- CMD-SHELL
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
networks:
- seaweed_internal
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3:
image: chrislusf/seaweedfs:4.51
entrypoint: ["weed"]
command:
- s3
- -filer=s3-core:8888
- -config=/run/secrets/s3_config
- -iam.config=/run/secrets/s3_iam_config
- -port=0
- -port.https=8334
- -cert.file=/run/secrets/s3_tls_cert
- -key.file=/run/secrets/s3_tls_key
- -cacert.file=/run/secrets/ca_crt
- -tlsVerifyClientCert=true
- -port.grpc=0
- -port.iceberg=0
- -port.lance=0
- -concurrentFileUploadLimit=10
- -concurrentUploadLimitMB=100
- -readerCacheSizeMB=64
- -autoCreateBucket=false
- -ip.bind=0.0.0.0
secrets:
- s3_config
- s3_iam_config
- s3_tls_cert
- s3_tls_key
- ca_crt
- s3_healthcheck_client_cert
- s3_healthcheck_client_key
depends_on:
s3-core:
condition: service_healthy
networks:
- default
- seaweed_internal
healthcheck:
test:
- CMD-SHELL
- "code=$$(curl --cacert /run/secrets/ca_crt --cert /run/secrets/s3_healthcheck_client_cert --key /run/secrets/s3_healthcheck_client_key -sS -o /dev/null -w '%{http_code}' https://s3:8334/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
interval: 3s
timeout: 2s
retries: 30
start_period: 10s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
s3-bootstrap:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/bootstrap_s3.py"]
environment:
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_bootstrap_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_bootstrap_client_key
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets:
- ca_crt
- s3_bootstrap_client_cert
- s3_bootstrap_client_key
- s3_admin_access_key_id
- s3_admin_secret_access_key
depends_on:
s3:
condition: service_healthy
restart: "no"
postgres-tls-init:
image: alpine:3.21.3
user: "0:0"
command:
- sh
- -c
- "cp /run/secrets/postgres_tls_cert /tls/server.crt && cp /run/secrets/postgres_tls_key /tls/server.key && chmod 0644 /tls/server.crt && chmod 0600 /tls/server.key && chown 70:70 /tls/server.crt /tls/server.key"
secrets:
- postgres_tls_cert
- postgres_tls_key
volumes:
- postgres_tls:/tls
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=8m
cap_drop:
- ALL
cap_add:
- CHOWN
security_opt:
- no-new-privileges:true
restart: "no"
migrate:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/migrate.py"]
environment:
BAO_ADDR: https://openbao:8200
BAO_CACERT: /run/secrets/ca_crt
BAO_ROLE_ID_FILE: /run/secrets/migrate_role_id
BAO_SECRET_ID_FILE: /run/secrets/migrate_secret_id
BAO_SECRET_PATH: messenger/migrate
WORKER_DB_PASSWORD_FILE: /run/secrets/worker_db_password
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_migrate_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_migrate_client_key
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
DB_POOL_MAX_SIZE: "2"
secrets:
- ca_crt
- migrate_role_id
- migrate_secret_id
- worker_db_password
- s3_migrate_client_cert
- s3_migrate_client_key
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
depends_on:
postgres:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
TLS_CERT_FILE: /run/secrets/api_tls_cert
TLS_KEY_FILE: /run/secrets/api_tls_key
TLS_CA_FILE: /run/secrets/ca_crt
TLS_CLIENT_CERT_FILE: /run/secrets/lb_tls_cert
BAO_ADDR: https://openbao:8200
BAO_CACERT: /run/secrets/ca_crt
BAO_ROLE_ID_FILE: /run/secrets/api_role_id
BAO_SECRET_ID_FILE: /run/secrets/api_secret_id
BAO_SECRET_PATH: messenger/api
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_api_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_api_client_key
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
ARGON2_MEMORY_KIB: "19456"
ARGON2_ITERATIONS: "2"
ARGON2_PARALLELISM: "1"
PASSWORD_HASH_CONCURRENCY: "4"
PASSWORD_HASH_WAIT_SECONDS: "0.1"
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
UPLOAD_TIMEOUT_SECONDS: "60"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX_SIZE: "8"
DB_POOL_TIMEOUT_SECONDS: "3"
DB_STATEMENT_TIMEOUT_MS: "5000"
DB_LOCK_TIMEOUT_MS: "2000"
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_api_client_cert
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_api_client_key
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- ca_crt
- api_tls_cert
- api_tls_key
- lb_tls_cert
- lb_tls_key
- wrong_client_cert
- wrong_client_key
- foreign_client_cert
- foreign_client_key
- api_role_id
- api_secret_id
- otel_api_client_cert
- otel_api_client_key
- s3_api_client_cert
- s3_api_client_key
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
depends_on:
migrate:
condition: service_completed_successfully
openbao:
condition: service_started
otel-collector:
condition: service_started
healthcheck:
test:
- CMD
- python3
- -c
- "import ssl,urllib.request; c=ssl.create_default_context(cafile='/run/secrets/ca_crt'); c.load_cert_chain('/run/secrets/lb_tls_cert','/run/secrets/lb_tls_key'); urllib.request.urlopen('https://127.0.0.1:8080/health/ready', context=c, timeout=2).read()"
interval: 3s
timeout: 3s
retries: 5
start_period: 10s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_LIMIT:-1.0}"
memory: "${API_MEMORY_LIMIT:-512M}"
stop_grace_period: 20s
restart: unless-stopped
publisher:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.publisher"]
environment:
BAO_ADDR: https://openbao:8200
BAO_CACERT: /run/secrets/ca_crt
BAO_ROLE_ID_FILE: /run/secrets/publisher_role_id
BAO_SECRET_ID_FILE: /run/secrets/publisher_secret_id
BAO_SECRET_PATH: messenger/publisher
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_publisher_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_publisher_client_key
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
DB_POOL_MAX_SIZE: "2"
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-publisher
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_publisher_client_cert
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_publisher_client_key
BROKER_CA_FILE: /run/secrets/ca_crt
BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_publisher_client_cert
BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_publisher_client_key
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- ca_crt
- publisher_role_id
- publisher_secret_id
- otel_publisher_client_cert
- otel_publisher_client_key
- rabbitmq_publisher_client_cert
- rabbitmq_publisher_client_key
- s3_publisher_client_cert
- s3_publisher_client_key
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
depends_on:
migrate:
condition: service_completed_successfully
openbao:
condition: service_started
rabbitmq:
condition: service_healthy
otel-collector:
condition: service_started
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
worker:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "-m", "src.worker"]
environment:
BAO_ADDR: https://openbao:8200
BAO_CACERT: /run/secrets/ca_crt
BAO_ROLE_ID_FILE: /run/secrets/worker_role_id
BAO_SECRET_ID_FILE: /run/secrets/worker_secret_id
BAO_SECRET_PATH: messenger/worker
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_worker_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_worker_client_key
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
MAX_IMAGE_PIXELS: "40000000"
DB_POOL_MAX_SIZE: "2"
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
APP_ENV: lab
OTEL_SERVICE_NAME: messenger-worker
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel_worker_client_cert
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel_worker_client_key
BROKER_CA_FILE: /run/secrets/ca_crt
BROKER_CLIENT_CERT_FILE: /run/secrets/rabbitmq_worker_client_cert
BROKER_CLIENT_KEY_FILE: /run/secrets/rabbitmq_worker_client_key
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-parentbased_traceidratio}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
secrets:
- ca_crt
- worker_role_id
- worker_secret_id
- otel_worker_client_cert
- otel_worker_client_key
- rabbitmq_worker_client_cert
- rabbitmq_worker_client_key
- s3_worker_client_cert
- s3_worker_client_key
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
depends_on:
migrate:
condition: service_completed_successfully
openbao:
condition: service_started
rabbitmq:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
otel-collector:
condition: service_started
deploy:
replicas: ${WORKER_REPLICAS:-1}
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
reconciler:
build:
context: .
dockerfile: Dockerfile
command: ["sh", "scripts/reconcile-loop.sh"]
environment:
BAO_ADDR: https://openbao:8200
BAO_CACERT: /run/secrets/ca_crt
BAO_ROLE_ID_FILE: /run/secrets/reconciler_role_id
BAO_SECRET_ID_FILE: /run/secrets/reconciler_secret_id
BAO_SECRET_PATH: messenger/reconciler
S3_ENDPOINT: https://s3:8334
S3_CA_FILE: /run/secrets/ca_crt
S3_CLIENT_CERT_FILE: /run/secrets/s3_reconciler_client_cert
S3_CLIENT_KEY_FILE: /run/secrets/s3_reconciler_client_key
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_STAGING_BUCKET: messenger-staging
S3_VARIANTS_BUCKET: messenger-variants
DB_POOL_MAX_SIZE: "2"
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
secrets:
- ca_crt
- reconciler_role_id
- reconciler_secret_id
- s3_reconciler_client_cert
- s3_reconciler_client_key
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=32m
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
depends_on:
migrate:
condition: service_completed_successfully
openbao:
condition: service_started
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.163.2
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
secrets:
- ca_crt
- otel_collector_tls_cert
- otel_collector_tls_key
- tempo_tls_cert
- loki_tls_cert
depends_on:
- tempo
- loki
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
prometheus:
image: prom/prometheus:v2.55.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --web.config.file=/etc/prometheus/web.yml
secrets:
- prometheus_tls_cert
- prometheus_tls_key
- ca_crt
- collector_prometheus_client_cert
- collector_prometheus_client_key
volumes:
- ./infra/otel/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/prometheus-web.yml:/etc/prometheus/web.yml:ro
- ./infra/otel/alerts.yml:/etc/prometheus/alerts.yml:ro
- prometheus_data:/prometheus
depends_on:
- otel-collector
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
loki:
image: grafana/loki:3.3.2
command: ["-config.file=/etc/loki/loki.yml"]
secrets:
- loki_tls_cert
- loki_tls_key
volumes:
- ./infra/otel/loki.yml:/etc/loki/loki.yml:ro
- loki_data:/loki
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.1
command: ["-config.file=/etc/tempo/tempo.yml"]
secrets:
- tempo_tls_cert
- tempo_tls_key
volumes:
- ./infra/otel/tempo.yml:/etc/tempo/tempo.yml:ro
- tempo_data:/var/tempo
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
grafana:
image: grafana/grafana:11.4.0
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
SSL_CERT_FILE: /run/secrets/ca_crt
secrets:
- grafana_admin_password
- ca_crt
volumes:
- grafana_data:/var/lib/grafana
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
depends_on:
- prometheus
- loki
- tempo
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
restart: unless-stopped
load_balancer:
image: haproxy:3.0.7-alpine
ports:
- "127.0.0.1:${APP_PORT:-8443}:8443"
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
secrets:
- ingress_pem
- ca_crt
- lb_client_pem
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
restart: unless-stopped
openbao:
image: quay.io/openbao/openbao:2.1.1
command: ["server", "-config=/bao/config.hcl"]
ports:
- "127.0.0.1:8200:8200"
volumes:
- openbao_data:/openbao/data
- ./infra/openbao/config.hcl:/bao/config.hcl:ro
secrets:
- openbao_tls_cert
- openbao_tls_key
- ca_crt
cap_drop:
- ALL
cap_add:
- SETGID
- SETUID
depends_on:
openbao-data-init:
condition: service_completed_successfully
security_opt:
- no-new-privileges:true
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
restart: unless-stopped
openbao-data-init:
image: alpine:3.21.3
user: "0:0"
command: ["sh", "-c", "mkdir -p /openbao/data && chown 100:1000 /openbao/data"]
volumes:
- openbao_data:/openbao/data
read_only: true
tmpfs:
- /tmp:rw,noexec,nosuid,size=8m
cap_drop:
- ALL
cap_add:
- CHOWN
security_opt:
- no-new-privileges:true
restart: "no"
secrets:
postgres_password:
file: ./secrets/postgres_password
worker_db_password:
file: ./secrets/worker_db_password
postgres_tls_cert:
file: ./secrets/pki/postgres.crt
postgres_tls_key:
file: ./secrets/pki/postgres.key
s3_bootstrap_client_cert:
file: ./secrets/pki/s3-bootstrap-client.crt
s3_bootstrap_client_key:
file: ./secrets/pki/s3-bootstrap-client.key
s3_healthcheck_client_cert:
file: ./secrets/pki/s3-healthcheck-client.crt
s3_healthcheck_client_key:
file: ./secrets/pki/s3-healthcheck-client.key
s3_api_client_cert:
file: ./secrets/pki/s3-api-client.crt
s3_api_client_key:
file: ./secrets/pki/s3-api-client.key
s3_worker_client_cert:
file: ./secrets/pki/s3-worker-client.crt
s3_worker_client_key:
file: ./secrets/pki/s3-worker-client.key
s3_publisher_client_cert:
file: ./secrets/pki/s3-publisher-client.crt
s3_publisher_client_key:
file: ./secrets/pki/s3-publisher-client.key
s3_migrate_client_cert:
file: ./secrets/pki/s3-migrate-client.crt
s3_migrate_client_key:
file: ./secrets/pki/s3-migrate-client.key
s3_reconciler_client_cert:
file: ./secrets/pki/s3-reconciler-client.crt
s3_reconciler_client_key:
file: ./secrets/pki/s3-reconciler-client.key
database_url:
file: ./secrets/database_url
s3_config:
file: ./secrets/s3.json
s3_iam_config:
file: ./secrets/s3_iam.json
s3_tls_cert:
file: ./secrets/pki/s3.crt
s3_tls_key:
file: ./secrets/pki/s3.key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
rabbitmq_password:
file: ./secrets/rabbitmq_password
rabbitmq_config:
file: ./secrets/rabbitmq.conf
rabbitmq_definitions:
file: ./secrets/rabbitmq_definitions.json
broker_url:
file: ./secrets/broker_url
broker_worker_url:
file: ./secrets/broker_worker_url
broker_publisher_url:
file: ./secrets/broker_publisher_url
grafana_admin_password:
file: ./secrets/grafana_admin_password
ca_crt:
file: ./secrets/pki/ca.crt
api_tls_cert:
file: ./secrets/pki/api.crt
api_tls_key:
file: ./secrets/pki/api.key
lb_tls_cert:
file: ./secrets/pki/lb-client.crt
lb_tls_key:
file: ./secrets/pki/lb-client.key
lb_client_pem:
file: ./secrets/pki/lb-client.pem
ingress_pem:
file: ./secrets/pki/ingress.pem
openbao_tls_cert:
file: ./secrets/pki/openbao.crt
openbao_tls_key:
file: ./secrets/pki/openbao.key
rabbitmq_tls_cert:
file: ./secrets/pki/rabbitmq.crt
rabbitmq_tls_key:
file: ./secrets/pki/rabbitmq.key
otel_collector_tls_cert:
file: ./secrets/pki/otel-collector.crt
otel_collector_tls_key:
file: ./secrets/pki/otel-collector.key
tempo_tls_cert:
file: ./secrets/pki/tempo.crt
tempo_tls_key:
file: ./secrets/pki/tempo.key
loki_tls_cert:
file: ./secrets/pki/loki.crt
loki_tls_key:
file: ./secrets/pki/loki.key
prometheus_tls_cert:
file: ./secrets/pki/prometheus.crt
prometheus_tls_key:
file: ./secrets/pki/prometheus.key
collector_prometheus_client_cert:
file: ./secrets/pki/collector-prometheus-client.crt
collector_prometheus_client_key:
file: ./secrets/pki/collector-prometheus-client.key
otel_api_client_cert:
file: ./secrets/pki/otel-api-client.crt
otel_api_client_key:
file: ./secrets/pki/otel-api-client.key
otel_worker_client_cert:
file: ./secrets/pki/otel-worker-client.crt
otel_worker_client_key:
file: ./secrets/pki/otel-worker-client.key
otel_publisher_client_cert:
file: ./secrets/pki/otel-publisher-client.crt
otel_publisher_client_key:
file: ./secrets/pki/otel-publisher-client.key
rabbitmq_api_client_cert:
file: ./secrets/pki/rabbitmq-api-client.crt
rabbitmq_api_client_key:
file: ./secrets/pki/rabbitmq-api-client.key
rabbitmq_worker_client_cert:
file: ./secrets/pki/rabbitmq-worker-client.crt
rabbitmq_worker_client_key:
file: ./secrets/pki/rabbitmq-worker-client.key
rabbitmq_publisher_client_cert:
file: ./secrets/pki/rabbitmq-publisher-client.crt
rabbitmq_publisher_client_key:
file: ./secrets/pki/rabbitmq-publisher-client.key
wrong_client_cert:
file: ./secrets/pki/wrong-client.crt
wrong_client_key:
file: ./secrets/pki/wrong-client.key
foreign_client_cert:
file: ./secrets/pki/foreign-client.crt
foreign_client_key:
file: ./secrets/pki/foreign-client.key
api_role_id:
file: ./secrets/openbao/api_role_id
api_secret_id:
file: ./secrets/openbao/api_secret_id
worker_role_id:
file: ./secrets/openbao/worker_role_id
worker_secret_id:
file: ./secrets/openbao/worker_secret_id
publisher_role_id:
file: ./secrets/openbao/publisher_role_id
publisher_secret_id:
file: ./secrets/openbao/publisher_secret_id
migrate_role_id:
file: ./secrets/openbao/migrate_role_id
migrate_secret_id:
file: ./secrets/openbao/migrate_secret_id
reconciler_role_id:
file: ./secrets/openbao/reconciler_role_id
reconciler_secret_id:
file: ./secrets/openbao/reconciler_secret_id
volumes:
postgres_data:
postgres_tls:
seaweed_data:
rabbitmq_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
openbao_data:
networks:
seaweed_internal:
internal: true
+26 -28
View File
@@ -1,37 +1,35 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия — lab7
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
Для каждой связи указаны используемые endpoint, protocol, identity и фактический способ проверки. Состояние относится к реализации в `compose.yaml`.
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| Связь | Фактический endpoint / protocol / identity | Проверка, срок действия | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Клиент → ingress/LB | `localhost:8443`, HTTPS; CA `secrets/pki/ca.crt`; ingress SAN `localhost`, `127.0.0.1`; внешний client certificate не требуется | Python HTTPS client проверяет CA и hostname; пользовательская идентичность — Bearer session; пять попыток на username и шестьдесят на source IP за минуту, затем 429/Retry-After | `haproxy.cfg`, `scripts/bootstrap-pki.py`, `tests/security.py`, `tests/login_rate_limit.py` |
| LB → API (все реплики) | `api:8080`, TLS 1.2+, mTLS; CA `ca.crt`; HAProxy cert URI SAN `spiffe://mtusi2026lab7/load-balancer`; API SAN `DNS:api` | HAProxy проверяет CA и DNS hostname; API проверяет клиентскую CA chain и точный URI SAN; leaf lifetime 397 дней | `haproxy.cfg`, `src/app.py`, `src/security.py`, `tests/internal_mtls.py`; `make test-internal-mtls` |
| API/worker → PostgreSQL | `postgres:5432`, PostgreSQL TLS over Compose network; CA verification (`sslmode=verify-full`); separate API and `messenger_worker` DB roles | Server certificate and hostname verified; worker role limited to attachment/outbox operations and columns | `compose.yaml`, `infra/postgres/pg_hba.conf`, `src/store.py`, `tests/worker_isolation.py` |
| API/worker/publisher/reconciler → S3 | `s3:8334`, HTTPS with mTLS; separate client certificates and access keys; CA verification | API reads originals/variants and lists originals as configured; worker reads/writes staging and writes originals/variants, but cannot list or read originals/variants; publisher/migrator have no object actions | `compose.yaml`, `scripts/prepare-secure-secrets.py`, `src/object_store.py`, `tests/worker_isolation.py` |
| Publisher/worker → broker | `rabbitmq:5671`, AMQPS with mTLS and CA/hostname verification; separate publisher and worker identities | Publisher can publish through the default exchange; worker consumes `attachments.process.v1` and is denied publishing; queue definitions are provisioned separately | `compose.yaml`, `src/jobs.py`, `scripts/prepare-secure-secrets.py`, `tests/worker_isolation.py` |
| API/worker/publisher → Collector | `otel-collector:4318`, OTLP/HTTP over HTTPS with mTLS; distinct client certificates and CA verification | Collector requires a client certificate from the lab CA; application uses TLS exporter settings | `compose.yaml`, `infra/otel/collector.yaml`, `src/telemetry.py` |
| Collector → telemetry backends | Prometheus scrape endpoint uses HTTPS/mTLS; Loki and Tempo use HTTPS with CA validation | Prometheus client certificate is checked by Collector; Collector validates backend CA | `infra/otel/collector.yaml`, `infra/otel/prometheus.yml`, `infra/otel/loki.yml`, `infra/otel/tempo.yml` |
| Grafana → data sources | Prometheus, Loki and Tempo HTTPS service endpoints | Grafana uses the local CA trust bundle; endpoints were queried from the Grafana container | `compose.yaml`, `infra/otel/grafana/provisioning/datasources/datasources.yml` |
| API/worker/agent → OpenBao | `https://openbao:8200`, CA verified; AppRole role_id + SecretID; KV v2 service-specific path | Per-service policy разрешает только `secret/data/messenger/<service>` read; SecretID TTL 24h, service token TTL 15m (max 1h) | `infra/openbao/config.hcl`, `scripts/bootstrap-openbao.py`, `src/config.py` |
| Оператор → OpenBao/Grafana/S3 admin | OpenBao опубликован только на `127.0.0.1:8200` по TLS; Grafana также на loopback; S3 admin используется bootstrap job внутри Compose network | OpenBao root token и unseal shares остаются у оператора; application services их не получают | `compose.yaml`, `infra/openbao/README.md`, `scripts/bootstrap-secrets.py` |
| API → Redis/Valkey, если используется | N/A: session source of truth — PostgreSQL; Redis/Valkey не выбран | Не применяется | `src/store.py`, `compose.yaml` |
| Replication/кластерные каналы, если добавлены | N/A: replicated database, broker cluster и multi-host transport не добавлены | Не применяется | `compose.yaml` |
| LB/API health checks | LB health request использует mTLS backend connection; API self-check использует loopback HTTPS и разрешённый LB certificate | API listener не опубликован на host; health endpoint не обходит mTLS | `haproxy.cfg`, `compose.yaml`, `src/app.py` |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
## Отрицательные проверки
## Обязательные отрицательные проверки
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
1. `make test-security BASE_URL=https://localhost:8443 CA_FILE=...` выполняет положительный TLS probe и проверку отказа пустого trust store. Hostname проверяется клиентским TLS context.
2. `make test-internal-mtls` проверяет API listener без client cert, с cert от foreign CA и с URI SAN другой службы; разрешённый LB cert должен пройти.
3. `make test-login-limit` проверяет 429 и положительный Retry-After после неудачных попыток.
4. `make test` проверяет ACL чат-участников, logout, сессии, сообщения и вложения; дополнительный `X-User-Id` не является источником identity после lab1.
5. `make test-openbao-acl` проверяет, что каждая AppRole читает только собственный KV path и получает 403 на остальные service paths.
6. `make test-worker-isolation` проверяет разрешённые и запрещённые операции worker в PostgreSQL, S3 и RabbitMQ.
7. `make test-worker-rotation ROOT_TOKEN_FILE=...` отзывает старый worker SecretID, проверяет отказ старой identity и завершает queued attachment с заменённой identity.
## Bootstrap и эксплуатация
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
В Git хранятся только конфигурация и скрипты. CA private key, leaf keys, root token, unseal shares и AppRole SecretIDs создаются локально и хранятся в исключённом каталоге `secrets/`. Оператор отвечает за CA/unseal; OpenBao использует persistent file storage, TLS listener и AppRole auth. Root token не монтируется приложению. После restart оператор unseal-ит OpenBao при необходимости; приложение повторно аутентифицируется AppRole при старте.
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
Redis и кластерные каналы не используются. После перезапуска OpenBao operator выполняет unseal; приложение не использует plaintext fallback при недоступном/sealed OpenBao.
+1
View File
@@ -0,0 +1 @@
{"argon2_iterations": 2, "argon2_memory_kib": 19456, "argon2_parallelism": 1, "burst_capacity_rejections": 6, "burst_requests": 12, "burst_successes": 6, "hash_slots_per_process": 4, "hash_wait_seconds": 0.1, "max_rss_platform_units": 162220, "peak_active_hashes": 4, "platform": "linux", "sequential_samples": 20, "verify_latency_ms": {"max": 32.157, "p50": 23.007, "p95": 28.288}}
+1
View File
@@ -0,0 +1 @@
{"dependencies": [{"name": "psycopg", "version": "3.2.10", "vulns": []}, {"name": "psycopg-pool", "version": "3.3.3", "vulns": []}, {"name": "redis", "version": "5.2.1", "vulns": []}, {"name": "boto3", "version": "1.35.99", "vulns": []}, {"name": "pillow", "version": "13.0.0", "vulns": []}, {"name": "pika", "version": "1.3.2", "vulns": []}, {"name": "opentelemetry-api", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-sdk", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-http", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation-psycopg", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-botocore", "version": "0.66b2", "vulns": []}, {"name": "psutil", "version": "6.1.1", "vulns": []}, {"name": "argon2-cffi", "version": "23.1.0", "vulns": []}, {"name": "opentelemetry-exporter-http-transport", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-common", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-exporter-otlp-proto-common", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-instrumentation", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-instrumentation-dbapi", "version": "0.66b2", "vulns": []}, {"name": "opentelemetry-proto", "version": "1.45.2", "vulns": []}, {"name": "opentelemetry-semantic-conventions", "version": "0.66b2", "vulns": []}, {"name": "psycopg-binary", "version": "3.2.10", "vulns": []}, {"name": "botocore", "version": "1.35.99", "vulns": []}, {"name": "googleapis-common-protos", "version": "1.76.0", "vulns": []}, {"name": "jmespath", "version": "1.1.0", "vulns": []}, {"name": "opentelemetry-propagator-aws-xray", "version": "1.0.2", "vulns": []}, {"name": "protobuf", "version": "7.36.2", "vulns": []}, {"name": "python-dateutil", "version": "2.9.0.post0", "vulns": []}, {"name": "requests", "version": "2.34.2", "vulns": []}, {"name": "charset-normalizer", "version": "3.5.2", "vulns": []}, {"name": "idna", "version": "3.20", "vulns": []}, {"name": "s3transfer", "version": "0.10.4", "vulns": []}, {"name": "urllib3", "version": "2.8.0", "vulns": []}, {"name": "wrapt", "version": "2.5.2", "vulns": []}, {"name": "certifi", "version": "2026.10.23", "vulns": []}, {"name": "six", "version": "1.17.0", "vulns": []}, {"name": "typing-extensions", "version": "4.16.0", "vulns": []}, {"name": "argon2-cffi-bindings", "version": "26.1.0", "vulns": []}, {"name": "cffi", "version": "2.1.1", "vulns": []}, {"name": "pycparser", "version": "3.12", "vulns": []}], "fixes": []}
+7
View File
@@ -0,0 +1,7 @@
{
"failed_attempts": 0,
"first_recovery_seconds": 3.463,
"observed_instances_before": 2,
"successful_writes": 108,
"result": "PASS"
}
+1
View File
@@ -0,0 +1 @@
{"https_ingress_healthy": true, "images_unchanged": true, "new_leaf_accepted": true, "new_leaf_sha256": "55e47bba381c871fa4c4dbdf34994526247be6429e37ae071199e73fa71d21ea", "old_leaf_rejected": true, "result": "PASS"}
+7
View File
@@ -0,0 +1,7 @@
{
"attachment_content_preserved": true,
"confirmed_message_preserved": true,
"existing_session_preserved": true,
"hook": "scripts/restart.sh",
"result": "PASS"
}
+58
View File
@@ -0,0 +1,58 @@
{
"fixture": "tests/fixtures/sample.png",
"invariants": {
"ambiguous_confirm_republishes_stable_job": true,
"confirmed_work_eventually_ready": true,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
"duplicate_effects_create_no_second_variant_set": true,
"stale_attempt_writes_no_effects": true,
"stale_pipeline_cannot_publish_current_metadata": true
},
"result": "PASS",
"scenarios": [
{
"deliveries": 2,
"duplicate_delivery_writes": 0,
"effective_object_writes": 4,
"scenario": "duplicate_delivery_after_commit_before_ack"
},
{
"attempts": 2,
"scenario": "crash_after_claim",
"terminal_status": "ready"
},
{
"attempts": 2,
"scenario": "crash_after_original_put",
"terminal_status": "ready"
},
{
"current_attempt": 2,
"scenario": "out_of_order_attempt_delivery",
"stale_attempt_extra_writes": 0
},
{
"current_pipeline_version": 2,
"published_variant_versions": [
"v2"
],
"scenario": "stale_pipeline_after_newer_version"
},
{
"outbox_recovered_after_restart": true,
"published_after_restart": 1,
"scenario": "publisher_crash_before_publish"
},
{
"deliveries": 2,
"same_message_id": true,
"scenario": "publish_confirm_lost_after_broker_accept"
},
{
"configured_timeout_seconds": 1,
"observed_seconds": 1.007,
"scenario": "transform_timeout_terminated"
}
],
"track": "2"
}
+1
View File
@@ -0,0 +1 @@
{"api_traces_scanned": 1, "leaks": 0, "log_entries_scanned": 11, "result": "PASS", "synthetic_values_checked": 6}
+1
View File
@@ -0,0 +1 @@
{"broker": {"broker_identity_connected": true, "publish_denied": true}, "database": {"attachments_read": true, "role": "messenger_worker", "users_read_denied": true}, "result": "PASS", "s3": {"messenger-attachments_list_denied": true, "messenger-attachments_read_denied": true, "messenger-attachments_write": true, "messenger-staging_list_denied": true, "messenger-staging_read": true, "messenger-staging_write": true, "messenger-variants_list_denied": true, "messenger-variants_read_denied": true, "messenger-variants_write": true, "temporary_objects_removed": true}}
+1
View File
@@ -0,0 +1 @@
{"attachment_state": ["queued", "ready"], "message_attachment_link_preserved": true, "old_worker_secret_id_revoked": true, "original_sha256_preserved": true, "replacement_worker_secret_id_authenticated": true, "result": "PASS"}
+38
View File
@@ -0,0 +1,38 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
option http-keep-alive
timeout connect 2s
timeout client 15s
timeout server 15s
timeout http-request 5s
timeout check 1s
retries 2
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
frontend messenger
bind :8443 ssl crt /run/secrets/ingress_pem
http-request set-header X-Real-IP %[src]
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk
http-check connect ssl sni api
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
option redispatch
default-server inter 1s fall 2 rise 2
server-template api 1-10 api:8080 check ssl verify required ca-file /run/secrets/ca_crt crt /run/secrets/lb_client_pem verifyhost api sni str(api) resolvers docker resolve-prefer ipv4 init-addr libc,none
+1 -1
View File
@@ -1,6 +1,6 @@
# Инфраструктура студента
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Конфигурации выбранных сервисов находятся здесь; основная топология — в корневом `compose.yaml`. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. Текущие OpenBao policies/bootstrap описаны в `openbao/`; OTel provisioning перенесён из lab6 в `otel/`.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
+11 -4
View File
@@ -1,7 +1,14 @@
# OpenBao — реализовать конфигурацию и политики
# OpenBao service configuration and bootstrap
Добавьте non-dev конфигурацию listener/storage/TLS, API/worker policies, bootstrap и rotation scripts. Credentials и unseal material создаются локально; здесь нет готового root token или private key.
`config.hcl` configures a TLS listener and persistent file storage; Compose runs OpenBao outside dev mode. `bootstrap-openbao.py` provisions KV v2 secrets and per-process AppRoles after operator init/unseal.
Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения.
The API, worker, publisher, migrator and reconciler use separate AppRole credentials and policies. Each policy can read only its own `secret/data/messenger/<service>` KV v2 path. Service tokens have a 15-minute TTL and a one-hour maximum TTL; AppRole SecretIDs expire after 24 hours. Application code never receives the operator root token.
Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md.
Operator workflow:
1. Generate local keys and bootstrap secrets with the scripts documented in the root README.
2. Start the `openbao` service, run `bao operator init`, and unseal it using the operator-held shares. Keep the initial root token outside the repository.
3. Run `python3 scripts/bootstrap-openbao.py --root-token-file <operator-only-file>`. It writes non-root AppRole IDs/SecretIDs under ignored `secrets/openbao/`; values are not printed.
4. Start application services. On a normal restart, the file backend retains OpenBao state and the operator unseals it if required. If a service SecretID expires, provision a replacement and restart that service.
The script provisions distinct paths and identities; secret values originate from local bootstrap files and are not committed. Root token and unseal shares must never be mounted into application containers.
+3
View File
@@ -0,0 +1,3 @@
path "secret/data/messenger/api" {
capabilities = ["read"]
}
+15
View File
@@ -0,0 +1,15 @@
ui = false
api_addr = "https://openbao:8200"
cluster_addr = "https://openbao:8201"
storage "file" {
path = "/openbao/data"
}
listener "tcp" {
address = "0.0.0.0:8200"
cluster_address = "0.0.0.0:8201"
tls_cert_file = "/run/secrets/openbao_tls_cert"
tls_key_file = "/run/secrets/openbao_tls_key"
tls_min_version = "tls12"
}
+3
View File
@@ -0,0 +1,3 @@
path "secret/data/messenger/worker" {
capabilities = ["read"]
}
+42
View File
@@ -0,0 +1,42 @@
groups:
- name: messenger-slo
interval: 10s
rules:
- record: messenger:api_5xx_ratio:rate5m
expr: |
(sum(rate(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0))
/
clamp_min(sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])), 1e-9)
- record: messenger:api_availability:ratio7d
expr: |
1 -
(sum(increase(messenger_http_requests_total{status_class="5xx",http_route!~"/health/.*"}[7d])) or vector(0))
/
clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[7d])), 1)
- alert: MessengerApiErrorBudgetBurn
expr: messenger:api_5xx_ratio:rate5m > 0.01
for: 2m
labels:
severity: warning
annotations:
summary: API error budget burn exceeds the 1% short-window budget
description: The rolling 5-minute business API 5xx ratio is above 1%.
runbook: Check API traces for the failing route, then inspect correlated logs and DB/S3 spans.
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_age_seconds) > 60 and max(messenger_queue_depth) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Oldest attachment job has waited for more than 60 seconds
description: Inspect worker availability, RabbitMQ health, and the worker trace for the oldest job.
runbook: Check queue depth and age, worker consuming logs, broker, then recover the worker and confirm resolved state.
- alert: MessengerDatabasePoolSaturation
expr: max(messenger_db_pool_requests_waiting) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Requests are waiting for a database connection
description: Compare DB waiters with request traces and PostgreSQL health.
runbook: Inspect DB spans and pool use; verify database health and connection limits.
+80
View File
@@ -0,0 +1,80 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
tls:
cert_file: /run/secrets/otel_collector_tls_cert
key_file: /run/secrets/otel_collector_tls_key
client_ca_file: /run/secrets/ca_crt
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: /run/secrets/otel_collector_tls_cert
key_file: /run/secrets/otel_collector_tls_key
client_ca_file: /run/secrets/ca_crt
processors:
memory_limiter:
check_interval: 1s
limit_mib: 384
spike_limit_mib: 96
batch:
timeout: 1s
send_batch_size: 512
send_batch_max_size: 1024
exporters:
prometheus:
endpoint: 0.0.0.0:9464
tls:
cert_file: /run/secrets/otel_collector_tls_cert
key_file: /run/secrets/otel_collector_tls_key
client_ca_file: /run/secrets/ca_crt
send_timestamps: true
resource_to_telemetry_conversion:
enabled: true
otlp_grpc/tempo:
endpoint: tempo:4317
tls:
ca_file: /run/secrets/ca_crt
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
otlp_http/loki:
endpoint: https://loki:3100/otlp
tls:
ca_file: /run/secrets/ca_crt
sending_queue:
enabled: true
queue_size: 256
num_consumers: 2
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 30s
service:
telemetry:
logs:
level: warn
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_grpc/tempo]
@@ -0,0 +1,174 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": false,
"graphTooltip": 1,
"panels": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 0},
"id": 1,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m]))", "legendFormat": "{{http_route}}", "refId": "A"}],
"title": "API requests per second",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 12, "y": 0},
"id": 2,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",http_route!~\"/health/.*\"}[5m])) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[5m])), 1e-9)", "legendFormat": "5xx / business requests", "refId": "A"}],
"title": "Business API 5xx ratio (5m)",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 7},
"id": 3,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "API latency percentiles from merged histogram buckets",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 7},
"id": 4,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p50", "refId": "A"},
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p95", "refId": "B"},
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_jobs_duration_seconds_bucket{job_type=\"attachment.process.v1\"}[5m])))", "legendFormat": "p99", "refId": "C"}
],
"title": "Worker job duration percentiles from histogram buckets",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 15},
"id": 5,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[5m]))", "legendFormat": "{{outcome}}", "refId": "A"}],
"title": "Jobs processed / failed per second",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 15},
"id": 6,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_queue_depth)", "legendFormat": "queued attachments", "refId": "A"}],
"title": "Durable attachment backlog",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 15},
"id": 7,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest queued job", "refId": "A"}],
"title": "Age of oldest queued attachment",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 22},
"id": 8,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_db_pool_connections_used)", "legendFormat": "used", "refId": "A"}, {"expr": "max(messenger_db_pool_requests_waiting)", "legendFormat": "waiting", "refId": "B"}],
"title": "Database pool use / waiters",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 22},
"id": 9,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "busy / capacity", "refId": "A"}],
"title": "Worker saturation",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 22},
"id": 10,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_process_memory_usage)", "legendFormat": "RSS", "refId": "A"}],
"title": "Process memory",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 29},
"id": 11,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "max(messenger_process_cpu_utilization)", "legendFormat": "CPU / core", "refId": "A"}],
"title": "Process CPU utilization",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"gridPos": {"h": 7, "w": 16, "x": 8, "y": 29},
"id": 12,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachments_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95 time to ready", "refId": "A"}],
"title": "Durable acceptance to attachment ready",
"type": "timeseries"
},
{
"datasource": {"type": "loki", "uid": "loki"},
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 36},
"id": 13,
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": true, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
"targets": [{"expr": "{service_name=~\"messenger-api|messenger-worker\"} | json", "refId": "A"}],
"title": "Structured API / worker logs",
"type": "logs"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0.9, "max": 1}, "overrides": []},
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 44},
"id": 14,
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
"targets": [{"expr": "messenger:api_availability:ratio7d", "legendFormat": "7-day availability; objective 99%", "refId": "A"}],
"title": "API SLI: 7-day rolling availability (99% SLO)",
"type": "timeseries"
}
],
"schemaVersion": 39,
"tags": ["messenger", "observability"],
"templating": {"list": []},
"time": {"from": "now-1h", "to": "now"},
"timezone": "browser",
"title": "Messenger observability",
"uid": "messenger-observability",
"version": 1
}
@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: true
editable: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,39 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: https://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: https://loki:3100
jsonData:
derivedFields:
- name: TraceID
matcherRegex: '"trace_id":"([a-f0-9]+)"'
url: "$${__value.raw}"
datasourceUid: tempo
editable: false
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: https://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-1h"
spanEndTimeShift: "1h"
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{$${__tags}} | json | trace_id="$${__trace.traceId}"'
serviceMap:
datasourceUid: prometheus
editable: false
+37
View File
@@ -0,0 +1,37 @@
auth_enabled: false
server:
http_listen_port: 3100
http_tls_config:
cert_file: /run/secrets/loki_tls_cert
key_file: /run/secrets/loki_tls_key
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
+3
View File
@@ -0,0 +1,3 @@
tls_server_config:
cert_file: /run/secrets/prometheus_tls_cert
key_file: /run/secrets/prometheus_tls_key
+20
View File
@@ -0,0 +1,20 @@
global:
scrape_interval: 10s
evaluation_interval: 10s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs:
- job_name: otel-collector
scheme: https
tls_config:
ca_file: /run/secrets/ca_crt
cert_file: /run/secrets/collector_prometheus_client_cert
key_file: /run/secrets/collector_prometheus_client_key
server_name: otel-collector
static_configs:
- targets: ["otel-collector:9464"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+36
View File
@@ -0,0 +1,36 @@
server:
http_listen_port: 3200
http_tls_config:
cert_file: /run/secrets/tempo_tls_cert
key_file: /run/secrets/tempo_tls_key
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
tls:
cert_file: /run/secrets/tempo_tls_cert
key_file: /run/secrets/tempo_tls_key
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: /run/secrets/tempo_tls_cert
key_file: /run/secrets/tempo_tls_key
ingester:
max_block_duration: 5m
trace_idle_period: 10s
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
+3
View File
@@ -0,0 +1,3 @@
local all all trust
hostssl all all all scram-sha-256
hostnossl all all all reject
+43
View File
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title varchar(100) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method varchar(8) NOT NULL,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key varchar(128) NOT NULL,
request_hash bytea NOT NULL,
response jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
+59
View File
@@ -0,0 +1,59 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename varchar(255) NOT NULL,
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
staging_key text NOT NULL UNIQUE,
job_id uuid NOT NULL UNIQUE,
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
lease_until timestamptz,
status varchar(16) NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(variants) = 'array'),
error_code varchar(64),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, ordinal)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
CREATE TABLE attachment_outbox (
id uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachments(job_id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
attempt integer NOT NULL CHECK (attempt >= 1),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
UNIQUE (job_id, attempt)
);
CREATE INDEX attachment_outbox_pending_idx
ON attachment_outbox (available_at, created_at)
WHERE published_at IS NULL;
CREATE TABLE reconciliation_incidents (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, incident_type)
);
+16
View File
@@ -0,0 +1,16 @@
ALTER TABLE attachment_outbox
DROP CONSTRAINT attachment_outbox_job_id_fkey;
CREATE TABLE attachment_pipeline_events (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
prior_job_id uuid NOT NULL,
job_id uuid NOT NULL,
prior_version integer NOT NULL CHECK (prior_version >= 1),
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
action varchar(16) NOT NULL CHECK (action = 'replay'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachment_pipeline_events_attachment_created_idx
ON attachment_pipeline_events (attachment_id, created_at);
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE attachment_outbox
ADD COLUMN traceparent varchar(55),
ADD COLUMN tracestate varchar(512);
+7
View File
@@ -0,0 +1,7 @@
CREATE TABLE login_rate_limits (
username_key bytea PRIMARY KEY,
window_started_at timestamptz NOT NULL,
attempts integer NOT NULL CHECK (attempts > 0)
);
CREATE INDEX login_rate_limits_window_idx ON login_rate_limits (window_started_at);
+2 -1
View File
@@ -1,2 +1,3 @@
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
# Development-only schema validation and dependency auditing tools.
openapi-spec-validator==0.7.2
pip-audit==2.10.1
+13
View File
@@ -0,0 +1,13 @@
psycopg[binary]==3.2.10
psycopg-pool==3.3.3
redis==5.2.1
boto3==1.35.99
Pillow==13.0.0
pika==1.3.2
opentelemetry-api==1.45.2
opentelemetry-sdk==1.45.2
opentelemetry-exporter-otlp-proto-http==1.45.2
opentelemetry-instrumentation-psycopg==0.66b2
opentelemetry-instrumentation-botocore==0.66b2
psutil==6.1.1
argon2-cffi==23.1.0
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
import argparse
import json
import os
from pathlib import Path
import ssl
from typing import Optional
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
ROLE_DIR = SECRET_DIR / "openbao"
SERVICES = {
"api": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
"worker": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
"publisher": ("DATABASE_URL", "BROKER_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
"migrate": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
"reconciler": ("DATABASE_URL", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
}
def main() -> None:
parser = argparse.ArgumentParser(description="Provision scoped OpenBao AppRoles for this local lab.")
parser.add_argument("--root-token-file", required=True, help="Operator-only file outside version control")
parser.add_argument("--ca-file", default=str(SECRET_DIR / "pki" / "ca.crt"))
parser.add_argument("--address", default="https://localhost:8200")
args = parser.parse_args()
root_token = Path(args.root_token_file).read_text(encoding="utf-8").strip()
if not root_token:
raise SystemExit("Root token file is empty.")
ROLE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
context = ssl.create_default_context(cafile=args.ca_file)
def request(method: str, path: str, payload: Optional[dict] = None) -> dict:
headers = {"X-Vault-Token": root_token, "Content-Type": "application/json"}
body = json.dumps(payload).encode("utf-8") if payload is not None else None
req = urllib.request.Request(
args.address.rstrip("/") + "/v1/" + path,
data=body,
headers=headers,
method=method,
)
try:
with urllib.request.urlopen(req, context=context, timeout=8) as response:
content = response.read(1024 * 1024)
return json.loads(content) if content else {}
except urllib.error.HTTPError as exc:
raise RuntimeError(f"OpenBao API request failed with HTTP {exc.code}") from None
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
raise RuntimeError(f"OpenBao API request failed ({type(exc).__name__})") from None
mounts = request("GET", "sys/mounts")
if "secret/" not in mounts.get("data", {}):
request("POST", "sys/mounts/secret", {"type": "kv", "options": {"version": "2"}})
auth = request("GET", "sys/auth")
if "approle/" not in auth.get("data", {}):
request("POST", "sys/auth/approle", {"type": "approle"})
local_values = {
"DATABASE_URL": (SECRET_DIR / "database_url").read_text(encoding="utf-8").strip(),
"WORKER_DATABASE_URL": (SECRET_DIR / "worker_database_url").read_text(encoding="utf-8").strip(),
"BROKER_URL": "",
"S3_ACCESS_KEY_ID": (SECRET_DIR / "s3_access_key_id").read_text(encoding="utf-8").strip(),
"S3_SECRET_ACCESS_KEY": (SECRET_DIR / "s3_secret_access_key").read_text(encoding="utf-8").strip(),
}
service_s3_files = {
"api": ("s3_access_key_id", "s3_secret_access_key"),
**{
service: (f"{service}_s3_access_key_id", f"{service}_s3_secret_access_key")
for service in ("worker", "publisher", "migrate", "reconciler")
},
}
for service, keys in SERVICES.items():
if service in ("worker", "publisher"):
local_values["BROKER_URL"] = (
SECRET_DIR / f"broker_{service}_url"
).read_text(encoding="utf-8").strip()
policy_name = f"messenger-{service}"
policy = (
f'path "secret/data/messenger/{service}" {{\n'
' capabilities = ["read"]\n'
'}\n'
)
request("PUT", f"sys/policies/acl/{policy_name}", {"policy": policy})
request(
"POST",
f"auth/approle/role/{service}",
{
"token_policies": [policy_name],
"token_ttl": "15m",
"token_max_ttl": "1h",
"secret_id_ttl": "24h",
"secret_id_num_uses": 0,
},
)
service_values = dict(local_values)
access_key_file, secret_key_file = service_s3_files[service]
service_values["S3_ACCESS_KEY_ID"] = (
SECRET_DIR / access_key_file
).read_text(encoding="utf-8").strip()
service_values["S3_SECRET_ACCESS_KEY"] = (
SECRET_DIR / secret_key_file
).read_text(encoding="utf-8").strip()
if service == "worker":
service_values["DATABASE_URL"] = local_values["WORKER_DATABASE_URL"]
request(
"POST",
f"secret/data/messenger/{service}",
{"data": {key: service_values[key] for key in keys}},
)
role = request("GET", f"auth/approle/role/{service}/role-id")
role_id = role.get("data", {}).get("role_id")
secret = request("POST", f"auth/approle/role/{service}/secret-id", {})
secret_id = secret.get("data", {}).get("secret_id")
if not role_id or not secret_id:
raise RuntimeError(f"OpenBao did not return credentials for the {service} role")
for suffix, value in (("role_id", role_id), ("secret_id", secret_id)):
path = ROLE_DIR / f"{service}_{suffix}"
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
print("OpenBao policies, scoped AppRoles and service credentials provisioned; values were not printed.")
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
PKI_DIR = ROOT / "secrets" / "pki"
def openssl(*args: str) -> None:
subprocess.run(
["openssl", *args],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def create_leaf(
work: Path,
name: str,
subject: str,
sans: str,
extended_usage: str,
ca_key: Path,
ca_cert: Path,
) -> tuple[Path, Path]:
key = PKI_DIR / f"{name}.key"
cert = PKI_DIR / f"{name}.crt"
request = work / f"{name}.csr"
extensions = work / f"{name}.ext"
openssl(
"req",
"-new",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
str(key),
"-out",
str(request),
"-subj",
subject,
)
extensions.write_text(
"basicConstraints=critical,CA:FALSE\n"
"keyUsage=critical,digitalSignature,keyEncipherment\n"
f"extendedKeyUsage={extended_usage}\n"
f"subjectAltName={sans}\n",
encoding="ascii",
)
openssl(
"x509",
"-req",
"-in",
str(request),
"-CA",
str(ca_cert),
"-CAkey",
str(ca_key),
"-CAcreateserial",
"-out",
str(cert),
"-days",
"397",
"-sha256",
"-extfile",
str(extensions),
)
os.chmod(key, 0o600)
os.chmod(cert, 0o644)
return cert, key
def bundle(cert: Path, key: Path, output: Path) -> None:
output.write_bytes(cert.read_bytes() + key.read_bytes())
os.chmod(output, 0o600)
def main() -> None:
if PKI_DIR.exists():
raise SystemExit("PKI files already exist; rotate them with an explicit rollover procedure.")
if not shutil.which("openssl"):
raise SystemExit("OpenSSL is required to generate local certificates.")
PKI_DIR.mkdir(mode=0o700, parents=True)
ca_key = PKI_DIR / "ca.key"
ca_cert = PKI_DIR / "ca.crt"
openssl(
"req",
"-x509",
"-newkey",
"rsa:3072",
"-nodes",
"-keyout",
str(ca_key),
"-out",
str(ca_cert),
"-days",
"3650",
"-sha256",
"-subj",
"/CN=mtusi2026lab7-local-ca",
"-addext",
"basicConstraints=critical,CA:TRUE",
"-addext",
"keyUsage=critical,keyCertSign,cRLSign",
)
os.chmod(ca_key, 0o600)
os.chmod(ca_cert, 0o644)
with tempfile.TemporaryDirectory(prefix="lab7-pki-") as directory:
work = Path(directory)
foreign_ca_key = PKI_DIR / "foreign-ca.key"
foreign_ca_cert = PKI_DIR / "foreign-ca.crt"
openssl(
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
str(foreign_ca_key),
"-out",
str(foreign_ca_cert),
"-days",
"365",
"-sha256",
"-subj",
"/CN=mtusi2026lab7-untrusted-ca",
"-addext",
"basicConstraints=critical,CA:TRUE",
"-addext",
"keyUsage=critical,keyCertSign,cRLSign",
)
os.chmod(foreign_ca_key, 0o600)
os.chmod(foreign_ca_cert, 0o644)
ingress_cert, ingress_key = create_leaf(
work,
"ingress",
"/CN=localhost",
"DNS:localhost,IP:127.0.0.1",
"serverAuth",
ca_key,
ca_cert,
)
create_leaf(
work,
"api",
"/CN=api",
"DNS:api,IP:127.0.0.1",
"serverAuth",
ca_key,
ca_cert,
)
client_cert, client_key = create_leaf(
work,
"lb-client",
"/CN=lab7-load-balancer",
"URI:spiffe://mtusi2026lab7/load-balancer",
"clientAuth",
ca_key,
ca_cert,
)
create_leaf(
work,
"wrong-client",
"/CN=lab7-untrusted-service",
"URI:spiffe://mtusi2026lab7/other-service",
"clientAuth",
ca_key,
ca_cert,
)
create_leaf(
work,
"foreign-client",
"/CN=lab7-foreign-service",
"URI:spiffe://mtusi2026lab7/load-balancer",
"clientAuth",
foreign_ca_key,
foreign_ca_cert,
)
create_leaf(
work,
"openbao",
"/CN=openbao",
"DNS:openbao,DNS:localhost,IP:127.0.0.1",
"serverAuth",
ca_key,
ca_cert,
)
bundle(ingress_cert, ingress_key, PKI_DIR / "ingress.pem")
bundle(client_cert, client_key, PKI_DIR / "lb-client.pem")
print("Local CA and leaf certificates created under secrets/pki with private permissions.")
if __name__ == "__main__":
main()
+254
View File
@@ -0,0 +1,254 @@
#!/usr/bin/env python3
import base64
import json
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
def main():
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
os.chmod(SECRET_DIR, 0o700)
names = (
"postgres_password",
"database_url",
"worker_db_password",
"worker_database_url",
"s3_access_key_id",
"s3_secret_access_key",
"worker_s3_access_key_id",
"worker_s3_secret_access_key",
"publisher_s3_access_key_id",
"publisher_s3_secret_access_key",
"migrate_s3_access_key_id",
"migrate_s3_secret_access_key",
"reconciler_s3_access_key_id",
"reconciler_s3_secret_access_key",
"s3_admin_access_key_id",
"s3_admin_secret_access_key",
"rabbitmq_password",
"rabbitmq.conf",
"rabbitmq_definitions.json",
"broker_url",
"broker_worker_url",
"broker_publisher_url",
"s3.json",
"s3_iam.json",
"grafana_admin_password",
)
existing = [name for name in names if (SECRET_DIR / name).exists()]
if existing:
raise SystemExit(
"Secret files already exist; refusing to rotate credentials without a migration: "
+ ", ".join(existing)
)
db_password = secrets.token_hex(24)
worker_db_password = secrets.token_urlsafe(32)
broker_password = secrets.token_urlsafe(32)
worker_broker_password = secrets.token_urlsafe(32)
publisher_broker_password = secrets.token_urlsafe(32)
values = {
"postgres_password": db_password,
"database_url": (
f"postgresql://messenger:{db_password}@postgres:5432/messenger"
"?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt"
),
"worker_db_password": worker_db_password,
"worker_database_url": (
f"postgresql://messenger_worker:{worker_db_password}@postgres:5432/messenger"
"?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt"
),
"s3_access_key_id": "mngr" + secrets.token_hex(8),
"s3_secret_access_key": secrets.token_urlsafe(40),
"worker_s3_access_key_id": "wrk" + secrets.token_hex(8),
"worker_s3_secret_access_key": secrets.token_urlsafe(40),
"publisher_s3_access_key_id": "pub" + secrets.token_hex(8),
"publisher_s3_secret_access_key": secrets.token_urlsafe(40),
"migrate_s3_access_key_id": "mig" + secrets.token_hex(8),
"migrate_s3_secret_access_key": secrets.token_urlsafe(40),
"reconciler_s3_access_key_id": "rec" + secrets.token_hex(8),
"reconciler_s3_secret_access_key": secrets.token_urlsafe(40),
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
"rabbitmq_password": broker_password,
"broker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f",
"broker_worker_url": f"amqps://worker:{worker_broker_password}@rabbitmq:5671/%2f",
"broker_publisher_url": f"amqps://publisher:{publisher_broker_password}@rabbitmq:5671/%2f",
"grafana_admin_password": secrets.token_urlsafe(32),
}
for name, value in values.items():
if name.startswith("broker_") and name != "broker_url":
descriptor = os.open(SECRET_DIR / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
continue
path = SECRET_DIR / name
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
descriptor = os.open(
SECRET_DIR / "rabbitmq.conf",
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write("default_user = messenger\n")
output.write(f"default_pass = {broker_password}\n")
output.write("listeners.tcp = none\n")
output.write("listeners.ssl.default = 5671\n")
output.write("ssl_options.cacertfile = /run/secrets/ca_crt\n")
output.write("ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n")
output.write("ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n")
output.write("ssl_options.verify = verify_peer\n")
output.write("ssl_options.fail_if_no_peer_cert = true\n")
output.write("ssl_options.versions.1 = tlsv1.2\n")
output.write("definitions.import_backend = local_filesystem\n")
output.write("definitions.local.path = /run/secrets/rabbitmq_definitions.json\n")
definitions = {
"users": [],
"vhosts": [{"name": "/"}],
"permissions": [],
"queues": [
{
"name": "attachments.process.dead",
"vhost": "/",
"durable": True,
"auto_delete": False,
"arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
},
{
"name": "attachments.process.v1",
"vhost": "/",
"durable": True,
"auto_delete": False,
"arguments": {
"x-queue-type": "quorum",
"x-quorum-initial-group-size": 1,
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": "attachments.process.dead",
},
},
],
}
for username, password, configure, write, read in (
("worker", worker_broker_password, r"^$", r"^$", r"^attachments\.process\.v1$"),
("publisher", publisher_broker_password, r"^$", r"^amq\.default$", r"^$"),
):
salt = secrets.token_bytes(4)
password_hash = __import__("hashlib").sha256(salt + password.encode("utf-8")).digest()
definitions["users"].append({
"name": username,
"password_hash": __import__("base64").b64encode(salt + password_hash).decode("ascii"),
"hashing_algorithm": "rabbit_password_hashing_sha256",
"tags": "",
})
definitions["permissions"].append({
"user": username,
"vhost": "/",
"configure": configure,
"write": write,
"read": read,
})
descriptor = os.open(
SECRET_DIR / "rabbitmq_definitions.json",
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(definitions, output, separators=(",", ":"))
output.write("\n")
os.chmod(SECRET_DIR / "rabbitmq_definitions.json", 0o644)
s3_config = {
"identities": [
{
"name": "bootstrap-admin",
"credentials": [
{
"accessKey": values["s3_admin_access_key_id"],
"secretKey": values["s3_admin_secret_access_key"],
}
],
"actions": ["Admin"],
},
{
"name": "messenger-api",
"credentials": [
{
"accessKey": values["s3_access_key_id"],
"secretKey": values["s3_secret_access_key"],
}
],
"actions": [
"Read:messenger-attachments",
"List:messenger-attachments",
"Read:messenger-variants",
"Write:messenger-staging",
],
},
{
"name": "messenger-worker",
"credentials": [{
"accessKey": values["worker_s3_access_key_id"],
"secretKey": values["worker_s3_secret_access_key"],
}],
"actions": [
"Read:messenger-staging",
"Write:messenger-staging",
"Write:messenger-attachments",
"Write:messenger-variants",
],
},
*[
{
"name": f"messenger-{service}",
"credentials": [{
"accessKey": values[f"{service}_s3_access_key_id"],
"secretKey": values[f"{service}_s3_secret_access_key"],
}],
"actions": [],
}
for service in ("publisher", "migrate")
],
{
"name": "messenger-reconciler",
"credentials": [{
"accessKey": values["reconciler_s3_access_key_id"],
"secretKey": values["reconciler_s3_secret_access_key"],
}],
"actions": [
f"{action}:{bucket}"
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants")
for action in ("Read", "Write", "List")
],
},
]
}
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(s3_config, output, separators=(",", ":"))
output.write("\n")
iam_config = {
"sts": {"signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")},
"defaultEffect": "Deny",
}
descriptor = os.open(SECRET_DIR / "s3_iam.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(iam_config, output, separators=(",", ":"))
output.write("\n")
role_dir = SECRET_DIR / "openbao"
role_dir.mkdir(mode=0o700, exist_ok=True)
for service in ("api", "worker", "publisher", "migrate", "reconciler"):
for suffix in ("role_id", "secret_id"):
path = role_dir / f"{service}_{suffix}"
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
os.close(descriptor)
print("Local secrets generated with owner-only permissions; values were not printed.")
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
import argparse
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
PKI_DIR = ROOT / "secrets" / "pki"
SERVICES = {
"rabbitmq": ("DNS:rabbitmq", "serverAuth"),
"s3": ("DNS:s3", "serverAuth"),
"postgres": ("DNS:postgres", "serverAuth"),
"otel-collector": ("DNS:otel-collector", "serverAuth"),
"tempo": ("DNS:tempo", "serverAuth"),
"loki": ("DNS:loki", "serverAuth"),
"prometheus": ("DNS:prometheus", "serverAuth"),
"collector-prometheus-client": ("URI:spiffe://mtusi2026lab7/collector", "clientAuth"),
"otel-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
"otel-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
"otel-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
"rabbitmq-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
"rabbitmq-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
"rabbitmq-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
"s3-api-client": ("URI:spiffe://mtusi2026lab7/api", "clientAuth"),
"s3-worker-client": ("URI:spiffe://mtusi2026lab7/worker", "clientAuth"),
"s3-publisher-client": ("URI:spiffe://mtusi2026lab7/publisher", "clientAuth"),
"s3-migrate-client": ("URI:spiffe://mtusi2026lab7/migrate", "clientAuth"),
"s3-reconciler-client": ("URI:spiffe://mtusi2026lab7/reconciler", "clientAuth"),
"s3-bootstrap-client": ("URI:spiffe://mtusi2026lab7/s3-bootstrap", "clientAuth"),
"s3-healthcheck-client": ("URI:spiffe://mtusi2026lab7/s3-healthcheck", "clientAuth"),
}
def openssl(*args: str) -> None:
subprocess.run(
["openssl", *args],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def main() -> None:
parser = argparse.ArgumentParser(description="Issue local short-lived mTLS service identities.")
parser.add_argument("--days", type=int, default=30)
args = parser.parse_args()
if not 1 <= args.days <= 90:
parser.error("service certificate lifetime must be between 1 and 90 days")
if not (PKI_DIR / "ca.key").is_file() or not (PKI_DIR / "ca.crt").is_file():
raise SystemExit("Create the local CA first with scripts/bootstrap-pki.py.")
if not shutil.which("openssl"):
raise SystemExit("OpenSSL is required to issue service certificates.")
missing = {
name: identity for name, identity in SERVICES.items()
if not (PKI_DIR / f"{name}.crt").exists()
}
if not missing:
raise SystemExit("All service certificates already exist; rotate them explicitly.")
with tempfile.TemporaryDirectory(prefix="lab7-service-pki-") as directory:
work = Path(directory)
for name, (san, usage) in missing.items():
key = PKI_DIR / f"{name}.key"
cert = PKI_DIR / f"{name}.crt"
csr = work / f"{name}.csr"
extension = work / f"{name}.ext"
openssl(
"req", "-new", "-newkey", "rsa:2048", "-nodes",
"-keyout", str(key), "-out", str(csr), "-subj", f"/CN={name}",
)
extension.write_text(
"basicConstraints=critical,CA:FALSE\n"
"keyUsage=critical,digitalSignature,keyEncipherment\n"
f"extendedKeyUsage={usage}\n"
f"subjectAltName={san}\n",
encoding="ascii",
)
openssl(
"x509", "-req", "-in", str(csr), "-CA", str(PKI_DIR / "ca.crt"),
"-CAkey", str(PKI_DIR / "ca.key"), "-CAcreateserial", "-out", str(cert),
"-days", str(args.days), "-sha256", "-extfile", str(extension),
)
os.chmod(key, 0o600)
os.chmod(cert, 0o644)
print(f"Issued {len(missing)} service certificates with {args.days}-day validity.")
if __name__ == "__main__":
main()
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
import base64
import hashlib
import json
import os
from pathlib import Path
import secrets
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets"
def read(name: str) -> str:
return (SECRET_DIR / name).read_text(encoding="utf-8").strip()
def write(name: str, content: str, replace: bool = False) -> None:
path = SECRET_DIR / name
flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if replace else os.O_EXCL)
descriptor = os.open(path, flags, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(content)
if not content.endswith("\n"):
output.write("\n")
def rabbit_user(name: str, password: str) -> dict:
salt = secrets.token_bytes(4)
digest = hashlib.sha256(salt + password.encode("utf-8")).digest()
return {
"name": name,
"password_hash": base64.b64encode(salt + digest).decode("ascii"),
"hashing_algorithm": "rabbit_password_hashing_sha256",
"tags": "",
}
def main() -> None:
os.chmod(SECRET_DIR, 0o700)
db_parts = urlsplit(read("database_url"))
query = dict(parse_qsl(db_parts.query, keep_blank_values=True))
query.update({"sslmode": "verify-full", "sslrootcert": "/run/secrets/ca_crt"})
write(
"database_url",
urlunsplit((db_parts.scheme, db_parts.netloc, db_parts.path, urlencode(query), "")),
replace=True,
)
worker_password_path = SECRET_DIR / "worker_db_password"
if not worker_password_path.exists():
write("worker_db_password", secrets.token_urlsafe(32))
worker_password = read("worker_db_password")
write(
"worker_database_url",
urlunsplit(
(
db_parts.scheme,
f"messenger_worker:{worker_password}@{db_parts.hostname}:{db_parts.port}",
db_parts.path,
urlencode(query),
"",
)
),
replace=(SECRET_DIR / "worker_database_url").exists(),
)
service_s3_credentials = {}
for service in ("worker", "publisher", "migrate", "reconciler"):
access_key_file = f"{service}_s3_access_key_id"
secret_key_file = f"{service}_s3_secret_access_key"
if not (SECRET_DIR / access_key_file).exists():
write(access_key_file, service[:3] + secrets.token_hex(8))
if not (SECRET_DIR / secret_key_file).exists():
write(secret_key_file, secrets.token_urlsafe(40))
service_s3_credentials[service] = (
read(access_key_file),
read(secret_key_file),
)
credentials = {}
users = []
permissions = []
for service, configure, write_access, read_access in (
("worker", r"^$", r"^$", r"^attachments\.process\.v1$"),
("publisher", r"^$", r"^amq\.default$", r"^$"),
):
password_file = f"rabbitmq_{service}_password"
password_path = SECRET_DIR / password_file
password = password_path.read_text(encoding="utf-8").strip() if password_path.exists() else secrets.token_urlsafe(32)
if not password_path.exists():
write(password_file, password)
credentials[service] = password
users.append(rabbit_user(service, password))
permissions.append({
"user": service,
"vhost": "/",
"configure": configure,
"write": write_access,
"read": read_access,
})
write(
f"broker_{service}_url",
f"amqps://{service}:{password}@rabbitmq:5671/%2f",
replace=(SECRET_DIR / f"broker_{service}_url").exists(),
)
definitions = {
"users": users,
"vhosts": [{"name": "/"}],
"permissions": permissions,
"queues": [
{
"name": "attachments.process.dead",
"vhost": "/",
"durable": True,
"auto_delete": False,
"arguments": {"x-queue-type": "quorum", "x-quorum-initial-group-size": 1},
},
{
"name": "attachments.process.v1",
"vhost": "/",
"durable": True,
"auto_delete": False,
"arguments": {
"x-queue-type": "quorum",
"x-quorum-initial-group-size": 1,
"x-dead-letter-exchange": "",
"x-dead-letter-routing-key": "attachments.process.dead",
},
},
],
}
definitions_path = SECRET_DIR / "rabbitmq_definitions.json"
if definitions_path.exists():
definitions_path.unlink()
write("rabbitmq_definitions.json", json.dumps(definitions, separators=(",", ":")))
os.chmod(definitions_path, 0o644)
config = (
"default_user = messenger\n"
f"default_pass = {read('rabbitmq_password')}\n"
"listeners.tcp = none\n"
"listeners.ssl.default = 5671\n"
"ssl_options.cacertfile = /run/secrets/ca_crt\n"
"ssl_options.certfile = /run/secrets/rabbitmq_tls_cert\n"
"ssl_options.keyfile = /run/secrets/rabbitmq_tls_key\n"
"ssl_options.verify = verify_peer\n"
"ssl_options.fail_if_no_peer_cert = true\n"
"ssl_options.versions.1 = tlsv1.2\n"
"definitions.import_backend = local_filesystem\n"
"definitions.local.path = /run/secrets/rabbitmq_definitions\n"
)
write("rabbitmq.conf", config, replace=True)
identities = [
{
"name": "bootstrap-admin",
"credentials": [{
"accessKey": read("s3_admin_access_key_id"),
"secretKey": read("s3_admin_secret_access_key"),
}],
"actions": ["Admin"],
},
{
"name": "messenger-api",
"credentials": [{
"accessKey": read("s3_access_key_id"),
"secretKey": read("s3_secret_access_key"),
}],
"actions": [
"Read:messenger-attachments",
"List:messenger-attachments",
"Read:messenger-variants",
"Write:messenger-staging",
],
},
{
"name": "messenger-worker",
"credentials": [{
"accessKey": service_s3_credentials["worker"][0],
"secretKey": service_s3_credentials["worker"][1],
}],
"actions": [
"Read:messenger-staging",
"Write:messenger-staging",
"Write:messenger-attachments",
"Write:messenger-variants",
],
},
*[
{
"name": f"messenger-{service}",
"credentials": [{
"accessKey": service_s3_credentials[service][0],
"secretKey": service_s3_credentials[service][1],
}],
"actions": [],
}
for service in ("publisher", "migrate")
],
{
"name": "messenger-reconciler",
"credentials": [{
"accessKey": service_s3_credentials["reconciler"][0],
"secretKey": service_s3_credentials["reconciler"][1],
}],
"actions": [
f"{action}:{bucket}"
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants")
for action in ("Read", "Write", "List")
],
},
]
write("s3.json", json.dumps({"identities": identities}, separators=(",", ":")), replace=True)
iam_config_path = SECRET_DIR / "s3_iam.json"
if not iam_config_path.exists():
iam_config = {
"sts": {
"signingKey": "base64-" + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
},
"defaultEffect": "Deny",
}
write("s3_iam.json", json.dumps(iam_config, separators=(",", ":")))
print("TLS endpoints and service-scoped broker credentials prepared locally; values were not printed.")
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
interval=${RECONCILE_INTERVAL_SECONDS:-900}
grace=${RECONCILE_GRACE_SECONDS:-3600}
case "$interval" in
''|*[!0-9]*)
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
exit 2
;;
esac
case "$grace" in
''|*[!0-9]*)
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
exit 2
;;
esac
if [ "$interval" -lt 60 ]; then
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
exit 2
fi
if [ "$grace" -lt 60 ]; then
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
exit 2
fi
while :
do
python3 src/reconcile.py --grace-seconds "$grace" --apply
sleep "$interval"
done
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
docker compose --env-file .env restart postgres rabbitmq s3-core s3
for service in postgres rabbitmq s3-core s3; do
ready=0
attempt=0
while [ "$attempt" -lt 90 ]; do
ready=1
containers=$(docker compose --env-file .env ps -q "$service")
[ -n "$containers" ] || ready=0
for container in $containers; do
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}running{{end}}' "$container")
[ "$status" = healthy ] || ready=0
done
[ "$ready" -eq 1 ] && break
attempt=$((attempt + 1))
sleep 1
done
[ "$ready" -eq 1 ] || {
printf 'Service %s did not become healthy after restart.\n' "$service" >&2
exit 1
}
done
docker compose --env-file .env restart api publisher worker reconciler
printf '%s\n' 'Restarted API and application stores without changing volumes.'
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
matches=0
target_container=
for container in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
matches=$((matches + 1))
target_container=$container
fi
done
if [ "$matches" -ne 1 ]; then
printf '%s\n' 'Could not resolve TARGET_INSTANCE_ID to exactly one API container.' >&2
exit 1
fi
docker stop --time 2 "$target_container" >/dev/null
+19 -7
View File
@@ -1,19 +1,31 @@
# Модель угроз — заполнить
# Модель угроз — lab7
## Активы и акторы
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
Активы: пароли пользователей, hashes сессий, сообщения и вложения, database/object-store/broker credentials, service identities, CA private key, OpenBao root token и unseal shares, telemetry. Акторы: внешний клиент, пользователь — участник чата, посторонний пользователь, скомпрометированный application service и локальный оператор.
Внешний атакующий может отправлять HTTPS запросы к loopback-published ingress. Пользователь действует только от своего Bearer session. Скомпрометированная служба получает только AppRole credential и секретный KV path, смонтированные этой службе. Оператор управляет PKI и OpenBao init/unseal. Модель не включает компрометацию Docker host, операционной системы или operator workstation.
## Границы доверия
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
Внешняя граница — клиент → HAProxy по HTTPS. Внутренний API boundary — HAProxy → API по mTLS с проверкой CA chain и allowlisted URI SAN. OpenBao доступен по TLS и выдаёт статические secrets сервисным AppRole identities. Схема сетевых связей и протоколы приведены в [матрице доверия](../contracts/trust-matrix.md).
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
| --- | --- | --- | --- | --- |
| TODO | TODO | TODO | TODO | TODO |
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
| Клиент в сети пытается перехватить пароль при входе или Bearer token | Credentials передаются только аутентифицированному endpoint | HTTPS ingress, CA и hostname validation; открытый HTTP endpoint не опубликован | `make test-security` с корректным и пустым trust store | Внешний endpoint использует локальную учебную CA, которую клиент должен получить доверенным способом |
| Клиент с сертификатом, подписанным доверенной CA, но с URI SAN другой службы, обращается к API | Только LB identity может установить соединение к API listener | mTLS + точное сравнение `spiffe://mtusi2026lab7/load-balancer` | `make test-internal-mtls` с identity `other-service` | Срок leaf ограничен 397 днями; автоматическая ротация не настроена |
| Клиент без сертификата или с cert чужой CA пытается обойти ingress и обратиться к API listener | API не обслуживает неподтверждённое внутреннее соединение | `CERT_REQUIRED`, CA verification, API порт не опубликован на host | `make test-internal-mtls` без cert и с foreign CA; прямого host-порта нет | Проверка выполняется внутри контейнерной сети лабораторного Compose |
| Пользователь вне чата запрашивает историю, metadata или content чужой attachment | Только член чата получает доступ к сообщениям и вложениям | Bearer session, ownership/membership checks, одинаковый 404 для чужих ресурсов | `make test` black-box сценарий ACL | Контракт не включает администрирование/редактирование состава участников |
| При утечке PostgreSQL snapshot атакующий подбирает пароль | Пароль не хранится plaintext и имеет per-user salt | Argon2id `m=19456 KiB, t=2, p=1`; сессионные токены хранятся как SHA-256 hashes | `make test-primitives` | Параметры следует повторно измерить на целевом hardware |
| Password spray пытается обойти лимит через несколько API-реплик и usernames | Серия неуспешных входов должна давать ограниченный ответ | PostgreSQL хранит fixed-window counters по SHA-256(username) и trusted source IP; sixth per-user attempt или 61st per-IP attempt в окне возвращает 429/Retry-After | `make test-login-limit` проверяет per-user threshold | Порог per-IP равен 60 попыткам за минуту |
| Скомпрометированный API/worker запрашивает operator или чужие сервисные secrets | Application identity не получает root token/unseal shares и чужие KV paths | AppRole per service, KV v2 path-specific read policy, short TTL | `make test-openbao-acl` подтверждает собственный path и 403 для остальных сервисов | Поведение после истечения TTL не тестировалось; KV values статические |
| Скомпрометированный worker пытается читать аккаунты/originals, перечислять buckets или публиковать свои jobs | Worker не получает DB users, S3 List/read originals/variants или broker publish privileges | Отдельная PostgreSQL-роль; S3 read/write на staging bucket, write на originals/variants; AMQP read только process queue | `make test-worker-isolation` проверяет grants, `AccessDenied` и broker `403`; `make test-worker-rotation` отзывает прежний SecretID и восстанавливает queued attachment | Проверен один worker и один queued recovery scenario на локальном Compose стенде |
| Ошибка приложения/инструментации записывает token или тело сообщения в telemetry | Credentials и пользовательские данные не экспортируются как telemetry attributes | OTel instrumentation задаёт низкокардинальные route attributes; проверка кодом/конфигурацией | `make test` не является проверкой утечек telemetry; отдельный grep/inspection не приложен | Отдельный telemetry redaction test не запускался |
## Lifecycle и инциденты
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
Оператор создаёт локальную CA и leaf сертификаты с `scripts/bootstrap-pki.py`; сертификаты и ключи лежат в ignored `secrets/pki/`. Root token/unseal shares создаются оператором при init и не передаются bootstrap скрипту через CLI arguments или app containers; `bootstrap-openbao.py` читает root token из локального файла, пишет policies и создаёт AppRole credentials под `secrets/openbao/`. Скрипт не печатает значения.
Ingress/API/OpenBao leaf certificates выпускаются на 397 дней; CA — на 3650 дней. Автоматический renewal/CA rollover не реализован. OpenBao AppRole SecretIDs имеют TTL 24 часа, сервисные tokens — 15 минут (max 1 час); приложение аутентифицируется и получает KV values при запуске. Ротация SecretID требует operator provisioning и перезапуска соответствующей службы. OpenBao state сохраняется в named volume; после рестарта оператор unseal-ит сервис по необходимости.
Runtime TLS проверен для внутренних сервисных каналов и telemetry backends; AppRole access isolation, worker ACL и восстановление после SecretID rotation зафиксированы отдельными сценариями. Автоматический rollover leaf-сертификатов не включён; оператор управляет их сроком и заменой.
+1
View File
@@ -0,0 +1 @@
"""Messenger API application package."""
+77
View File
@@ -0,0 +1,77 @@
import hashlib
import hmac
import logging
import os
import signal
import ssl
import threading
from functools import partial
from http.server import ThreadingHTTPServer
from .config import APP_HOST, APP_PORT
from .handlers import MessengerHandler
from .security import allowed_client_identity
from .store import Store
from .telemetry import initialize, start_runtime_metrics
LOGGER = logging.getLogger("messenger")
class MutualTLSServer(ThreadingHTTPServer):
def __init__(self, address, handler):
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.minimum_version = ssl.TLSVersion.TLSv1_2
context.load_cert_chain(os.environ["TLS_CERT_FILE"], os.environ["TLS_KEY_FILE"])
context.load_verify_locations(cafile=os.environ["TLS_CA_FILE"])
context.verify_mode = ssl.CERT_REQUIRED
self.tls_context = context
client_cert_pem = open(os.environ["TLS_CLIENT_CERT_FILE"], encoding="ascii").read()
client_cert_der = ssl.PEM_cert_to_DER_cert(client_cert_pem)
self.client_certificate_fingerprint = hashlib.sha256(client_cert_der).digest()
super().__init__(address, handler)
def get_request(self):
raw_socket, address = super().get_request()
try:
connection = self.tls_context.wrap_socket(raw_socket, server_side=True)
peer_certificate = connection.getpeercert(binary_form=True)
fingerprint = hashlib.sha256(peer_certificate or b"").digest()
if (
not peer_certificate
or not hmac.compare_digest(fingerprint, self.client_certificate_fingerprint)
or not allowed_client_identity(connection.getpeercert())
):
connection.close()
raise ssl.SSLError("client certificate identity is not authorized")
return connection, address
except (OSError, ssl.SSLError):
raw_socket.close()
raise
def run_server():
store = Store()
store.open()
start_runtime_metrics(store)
try:
handler = partial(MessengerHandler, store=store)
server = MutualTLSServer((APP_HOST, APP_PORT), handler)
def handle_shutdown(signum, frame):
LOGGER.info("received signal %s, shutting down gracefully", signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
try:
server.serve_forever()
finally:
server.server_close()
finally:
store.close()
def main():
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-api"))
run_server()
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
import logging
import os
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
LOGGER = logging.getLogger("messenger.s3.bootstrap")
def migrate_legacy_namespaces(client, source_bucket: str, targets: dict[str, str]) -> int:
copied = 0
for prefix, target_bucket in targets.items():
pages = client.get_paginator("list_objects_v2").paginate(
Bucket=source_bucket,
Prefix=prefix,
)
for page in pages:
for item in page.get("Contents", []):
key = item["Key"]
try:
target = client.head_object(Bucket=target_bucket, Key=key)
except ClientError as exc:
if exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 404:
raise
source = client.get_object(Bucket=source_bucket, Key=key)
body = source["Body"]
try:
content = body.read()
finally:
body.close()
if len(content) != item["Size"]:
raise RuntimeError(f"Legacy object read size mismatch for {key}")
client.put_object(
Bucket=target_bucket,
Key=key,
Body=content,
ContentLength=len(content),
ContentType=source.get("ContentType", "application/octet-stream"),
Metadata=source.get("Metadata", {}),
)
target = client.head_object(Bucket=target_bucket, Key=key)
copied += 1
if (
target["ContentLength"] != item["Size"]
or target.get("ETag") != item.get("ETag")
):
raise RuntimeError(f"Legacy object copy verification failed for {key}")
return copied
def secret(name: str) -> str:
direct = os.getenv(name)
if direct:
return direct
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError(f"Set {name} or {name}_FILE")
def main():
client = boto3.client(
"s3",
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
region_name="us-east-1",
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
verify=os.getenv("S3_CA_FILE", True),
config=Config(
connect_timeout=2,
read_timeout=5,
retries={"max_attempts": 3},
client_cert=(os.getenv("S3_CLIENT_CERT_FILE"), os.getenv("S3_CLIENT_KEY_FILE")),
),
)
buckets = (
os.getenv("S3_BUCKET", "messenger-attachments"),
os.getenv("S3_STAGING_BUCKET", "messenger-staging"),
os.getenv("S3_VARIANTS_BUCKET", "messenger-variants"),
)
for bucket in buckets:
try:
client.head_bucket(Bucket=bucket)
except ClientError as exc:
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
if status != 404:
raise
try:
client.create_bucket(Bucket=bucket)
except ClientError as create_error:
if create_error.response.get("Error", {}).get("Code") != "BucketAlreadyOwnedByYou":
raise
client.put_bucket_acl(Bucket=bucket, ACL="private")
copied = migrate_legacy_namespaces(
client,
buckets[0],
{
"staging/": buckets[1],
"variants/": buckets[2],
},
)
LOGGER.info("private S3 buckets bootstrapped; legacy objects copied=%d", copied)
if __name__ == "__main__":
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
main()
+126
View File
@@ -0,0 +1,126 @@
import json
import os
import re
import ssl
import socket
import uuid
from pathlib import Path
import threading
import urllib.error
import urllib.request
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.getenv("S3_REGION", "us-east-1")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_STAGING_BUCKET = os.getenv("S3_STAGING_BUCKET", "messenger-staging")
S3_VARIANTS_BUCKET = os.getenv("S3_VARIANTS_BUCKET", "messenger-variants")
_BAO_LOCK = threading.Lock()
_BAO_CACHE = {}
def _read_secret_file(path_variable: str) -> str:
file_path = os.getenv(path_variable)
if not file_path:
raise RuntimeError(f"Set {path_variable}")
return Path(file_path).read_text(encoding="utf-8").strip()
def _bao_secrets() -> dict:
with _BAO_LOCK:
if "values" in _BAO_CACHE:
return _BAO_CACHE["values"]
address = os.getenv("BAO_ADDR")
ca_file = os.getenv("BAO_CACERT")
role_id = _read_secret_file("BAO_ROLE_ID_FILE")
secret_id = _read_secret_file("BAO_SECRET_ID_FILE")
secret_path = os.getenv("BAO_SECRET_PATH")
if not address or not ca_file or not secret_path:
raise RuntimeError("OpenBao service authentication requires BAO_ADDR, BAO_CACERT and BAO_SECRET_PATH")
context = ssl.create_default_context(cafile=ca_file)
def request(path: str, payload: dict, token: str = "", method: str = "") -> dict:
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
data = json.dumps(payload).encode("utf-8") if payload else None
req = urllib.request.Request(
address.rstrip("/") + "/v1/" + path,
data=data,
headers=headers,
method=method or None,
)
try:
with urllib.request.urlopen(req, context=context, timeout=5) as response:
return json.loads(response.read(1024 * 1024))
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
raise RuntimeError(f"OpenBao request failed ({type(exc).__name__})") from None
auth = request("auth/approle/login", {"role_id": role_id, "secret_id": secret_id})
token = auth.get("auth", {}).get("client_token")
if not token:
raise RuntimeError("OpenBao AppRole login returned no service token")
response = request(f"secret/data/{secret_path.lstrip('/')}", {}, token, "GET")
values = response.get("data", {}).get("data")
if not isinstance(values, dict):
raise RuntimeError("OpenBao returned an invalid secret payload")
_BAO_CACHE["values"] = values
return values
def secret_value(name: str, default: str = "") -> str:
if os.getenv("BAO_ROLE_ID_FILE") or os.getenv("BAO_ROLE_ID"):
values = _bao_secrets()
if name in values and isinstance(values[name], str):
return values[name]
raise RuntimeError(f"OpenBao path does not contain {name}")
value = os.getenv(name)
if value:
return value
file_path = os.getenv(name + "_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
if default:
return default
raise RuntimeError(f"Set {name} or {name}_FILE")
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
if IDEMPOTENCY_TTL_SECONDS < 86400:
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
raise RuntimeError("Database timeouts must be positive")
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
raise RuntimeError("Database pool settings must be positive")
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
raise RuntimeError("HTTP limits must be positive")
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
if MAX_CONCURRENT_UPLOADS < 1:
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
if UPLOAD_TIMEOUT_SECONDS < 1:
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
+26
View File
@@ -0,0 +1,26 @@
from typing import Dict, Optional
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class CacheUnavailableError(Exception):
pass
class IdempotencyConflictError(Exception):
pass
class AttachmentNotReadyError(Exception):
pass
class ObjectMissingError(Exception):
pass
+653
View File
@@ -0,0 +1,653 @@
import base64
import hashlib
import json
import logging
import threading
import time
import tempfile
import uuid
import warnings
import zlib
from http.server import BaseHTTPRequestHandler
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
import unicodedata
import psycopg
from opentelemetry import propagate
from opentelemetry.instrumentation.utils import suppress_instrumentation
from opentelemetry.trace import SpanKind, Status, StatusCode
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, UnidentifiedImageError
from psycopg_pool import PoolTimeout
from .config import (
IDEMPOTENCY_KEY_RE,
INSTANCE_ID,
MAX_CONCURRENT_UPLOADS,
MAX_IMAGE_PIXELS,
MAX_INFLIGHT_REQUESTS,
MAX_REQUEST_BODY_BYTES,
MAX_UPLOAD_BYTES,
UPLOAD_TIMEOUT_SECONDS,
USER_RE,
)
from .errors import (
ApiError,
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .security import PasswordHashCapacityError, hash_password, isoformat, valid_uuid
from .store import Store
from .telemetry import REQUEST_ID, record_http_request, tracer
LOGGER = logging.getLogger("messenger.http")
class RequestGuard:
def __init__(self):
self.lock = threading.Lock()
self.inflight = 0
def acquire(self):
with self.lock:
if self.inflight >= MAX_INFLIGHT_REQUESTS:
raise ApiError(503, "unavailable", "Server is busy")
self.inflight += 1
def release(self):
with self.lock:
self.inflight = max(0, self.inflight - 1)
REQUEST_GUARD = RequestGuard()
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
class MessengerHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server, store: Store):
self.store = store
super().__init__(request, client_address, server)
def setup(self):
super().setup()
self.connection.settimeout(15)
def log_message(self, format: str, *args):
return
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
path = urlsplit(self.path).path
route = self._route_template(path)
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
parent_context = propagate.extract({key.lower(): value for key, value in self.headers.items()})
started = time.perf_counter()
token = REQUEST_ID.set(request_id)
self._response_status = 500
if route.startswith("/health/"):
try:
self._dispatch_request(method, request_id)
finally:
record_http_request(method, route, self._response_status, time.perf_counter() - started)
REQUEST_ID.reset(token)
return
with tracer().start_as_current_span(
f"{method} {route}",
context=parent_context,
kind=SpanKind.SERVER,
attributes={
"http.request.method": method,
"http.route": route,
"http.request_id": request_id,
},
) as span:
try:
self._dispatch_request(method, request_id)
finally:
duration = time.perf_counter() - started
span.set_attribute("http.response.status_code", self._response_status)
if self._response_status >= 500:
span.set_status(Status(StatusCode.ERROR))
record_http_request(method, route, self._response_status, duration)
REQUEST_ID.reset(token)
@staticmethod
def _route_template(path: str) -> str:
if path in ("/health/live", "/health/ready", "/api/v1/users", "/api/v1/auth/sessions",
"/api/v1/auth/sessions/current", "/api/v1/users/me", "/api/v1/chats"):
return path
if path.startswith("/api/v1/chats/") and path.endswith("/attachments"):
return "/api/v1/chats/{chat_id}/attachments"
if path.startswith("/api/v1/chats/") and path.endswith("/messages"):
return "/api/v1/chats/{chat_id}/messages"
if path.startswith("/api/v1/chats/"):
return "/api/v1/chats/{chat_id}"
if path.startswith("/api/v1/attachments/") and path.endswith("/content"):
return "/api/v1/attachments/{attachment_id}/content"
if path.startswith("/api/v1/attachments/"):
return "/api/v1/attachments/{attachment_id}"
return "unmatched"
def _dispatch_request(self, method: str, request_id: str):
acquired = False
try:
REQUEST_GUARD.acquire()
acquired = True
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == "/health/live" and method == "GET":
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/health/ready" and method == "GET":
try:
with suppress_instrumentation():
self.store.ready()
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
raise ApiError(503, "unavailable", "A required storage service is unavailable")
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/api/v1/users" and method == "POST":
self._handle_register(request_id)
elif path == "/api/v1/auth/sessions" and method == "POST":
self._handle_create_session(request_id)
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
self._handle_revoke_session(request_id)
elif path == "/api/v1/users/me" and method == "GET":
self._send_json(200, self._get_actor(request_id), request_id)
elif path == "/api/v1/chats" and method == "POST":
self._handle_create_chat(request_id)
elif path == "/api/v1/chats" and method == "GET":
self._handle_list_chats(query, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
chat_id = path.rsplit("/", 2)[1]
self._handle_upload_attachment(chat_id, query, request_id)
elif path.startswith("/api/v1/attachments/") and method == "GET":
attachment_path = path.removeprefix("/api/v1/attachments/")
if attachment_path.endswith("/content"):
attachment_id = attachment_path.removesuffix("/content")
self._handle_download_attachment(attachment_id, query, request_id)
else:
self._handle_get_attachment(attachment_path, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
chat_id = path.rsplit("/", 2)[1]
if method == "POST":
self._handle_create_message(chat_id, request_id)
else:
self._handle_list_messages(chat_id, query, request_id)
elif path.startswith("/api/v1/chats/") and method == "GET":
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
else:
raise ApiError(404, "not_found", "Object not found")
except ApiError as exc:
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
except CacheUnavailableError:
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
except PasswordHashCapacityError:
self._send_error(
503,
"unavailable",
"Password verification capacity is busy",
request_id,
{"Retry-After": "1"},
)
except IdempotencyConflictError:
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
except AttachmentNotReadyError:
self._send_error(409, "conflict", "Attachment is not ready", request_id)
except ObjectMissingError:
LOGGER.error("attachment object missing for request %s", request_id)
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
except (BotoCoreError, ClientError):
LOGGER.warning("object storage unavailable for request %s", request_id)
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
self._send_error(503, "unavailable", "Database is unavailable", request_id)
except psycopg.errors.UniqueViolation:
self._send_error(409, "conflict", "Username already exists", request_id)
except psycopg.errors.ForeignKeyViolation:
self._send_error(400, "invalid_request", "Unknown member", request_id)
except (BrokenPipeError, ConnectionResetError, TimeoutError):
self._set_response_status(499)
self.close_connection = True
LOGGER.info("client disconnected before response for request %s", request_id)
except Exception as exc:
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
self._send_error(500, "internal_error", "Internal server error", request_id)
finally:
if acquired:
REQUEST_GUARD.release()
@staticmethod
def _trace_context():
carrier = {}
propagate.inject(carrier)
return {"traceparent": carrier.get("traceparent"), "tracestate": carrier.get("tracestate")}
def _set_response_status(self, status: int) -> None:
self._response_status = status
def _read_json(self, request_id: str) -> Any:
if self.headers.get_content_type() != "application/json":
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Missing Content-Length")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size > MAX_REQUEST_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Request body too large")
raw = self.rfile.read(size)
if not raw:
raise ApiError(400, "invalid_request", "Empty body")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise ApiError(400, "invalid_request", "Malformed JSON")
def _get_actor(self, request_id: str) -> Dict[str, Any]:
header = self.headers.get("Authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not token or " " in token:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
user = self.store.session_user(token)
if user is None:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def _handle_register(self, request_id: str):
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
username = body["username"]
display_name = body["display_name"]
password = body["password"]
if not isinstance(username, str) or not USER_RE.fullmatch(username):
raise ApiError(400, "invalid_request", "Invalid username")
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
raise ApiError(400, "invalid_request", "Invalid display_name")
if not isinstance(password, str) or not (12 <= len(password) <= 128):
raise ApiError(400, "invalid_request", "Invalid password")
user = self.store.create_user(username, display_name, hash_password(password))
self._send_json(201, user, request_id)
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
value = self.headers.get("Authorization", "")
scheme, separator, encoded = value.partition(" ")
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
if scheme.lower() != "basic" or not separator or not encoded:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
if not USER_RE.fullmatch(username) or not 12 <= len(password) <= 128:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
return username, password
def _handle_create_session(self, request_id: str):
username, password = self._basic_credentials(request_id)
source_ip = self.headers.get("X-Real-IP", self.client_address[0])
retry_after = self.store.consume_login_attempt(username, source_ip)
if retry_after is not None:
raise ApiError(
429,
"rate_limited",
"Too many login attempts",
{"Retry-After": str(retry_after)},
)
user = self.store.authenticate(username, password)
if user is None:
raise ApiError(
401,
"unauthorized",
"Invalid credentials",
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
self.store.clear_login_attempts(username)
token, expires_at = self.store.create_session(user["id"])
self._send_json(
201,
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
request_id,
)
def _handle_revoke_session(self, request_id: str):
self._get_actor(request_id)
token = self.headers["Authorization"].partition(" ")[2]
if not self.store.revoke_session(token):
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
self._set_response_status(204)
self.send_response(204)
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.end_headers()
def _handle_create_chat(self, request_id: str):
actor = self._get_actor(request_id)
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
raise ApiError(400, "invalid_request", "Expected title and member_ids")
title, member_ids = body["title"], body["member_ids"]
if not isinstance(title, str) or not title.strip() or len(title) > 100:
raise ApiError(400, "invalid_request", "Invalid title")
if (
not isinstance(member_ids, list)
or not 1 <= len(member_ids) <= 100
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
or len(member_ids) != len(set(member_ids))
or actor["id"] not in member_ids
):
raise ApiError(400, "invalid_request", "Invalid member_ids")
chat = self.store.create_chat(title, member_ids, actor["id"])
self._send_json(201, chat, request_id)
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
def _handle_get_chat(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat = self.store.get_chat(chat_id, actor["id"])
if chat is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat, request_id)
def _handle_create_message(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
idempotency_key = self.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
text = body["text"]
attachment_ids = body.get("attachment_ids", [])
if (
not isinstance(text, str)
or len(text) > 4000
or not isinstance(attachment_ids, list)
or len(attachment_ids) > 10
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
or len(attachment_ids) != len(set(attachment_ids))
or (not text.strip() and not attachment_ids)
):
raise ApiError(400, "invalid_request", "Invalid message text")
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
canonical_body = json.dumps(
{"text": text, "attachment_ids": attachment_ids},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
message = self.store.create_idempotent_message(
chat_id,
actor["id"],
text,
attachment_ids,
idempotency_key,
canonical_body,
)
if message is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(201, message, request_id)
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
if declared_type not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
length = self.headers.get("Content-Length")
if length is None or self.headers.get("Transfer-Encoding"):
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size <= 0:
raise ApiError(415, "unsupported_media_type", "Image is empty")
if size > MAX_UPLOAD_BYTES:
self.close_connection = True
remaining = min(size, MAX_UPLOAD_BYTES + 1)
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
while remaining:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
break
self.connection.settimeout(min(15, remaining_time))
try:
chunk = self.rfile.read(min(64 * 1024, remaining))
except (TimeoutError, OSError):
LOGGER.info("stopped bounded drain of oversized upload")
break
if not chunk:
break
remaining -= len(chunk)
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
digest = hashlib.sha256()
bytes_read = 0
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
try:
while bytes_read < size:
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise ApiError(408, "invalid_request", "Image upload timed out")
self.connection.settimeout(min(15, remaining_time))
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
if not chunk:
raise ApiError(400, "invalid_request", "Incomplete image body")
content.write(chunk)
digest.update(chunk)
bytes_read += len(chunk)
except (TimeoutError, OSError):
content.close()
raise ApiError(400, "invalid_request", "Incomplete image body") from None
try:
content.seek(0)
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
if actual_type is None or actual_type != declared_type:
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
width, height = image.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
image.verify()
content.seek(0)
with IMAGE_DECODE_GUARD:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(content) as image:
image.load()
content.seek(0)
return content, actual_type, size, digest.hexdigest()
except ApiError:
content.close()
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
content.close()
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
content.close()
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
@staticmethod
def _valid_filename(filename: str) -> bool:
return (
1 <= len(filename) <= 255
and not any(char in ("/", "\\", "\x00") for char in filename)
and not any(unicodedata.category(char) == "Cc" for char in filename)
)
def _handle_upload_attachment(
self, chat_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
if not self.store.can_access_chat(chat_id, actor["id"]):
raise ApiError(404, "not_found", "Chat not found")
filenames = query.get("filename", [])
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
raise ApiError(400, "invalid_request", "Invalid filename")
if not UPLOAD_GUARD.acquire(blocking=False):
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
try:
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
try:
attachment = self.store.create_attachment(
chat_id,
actor["id"],
filenames[0],
content_type,
content,
size,
digest,
**self._trace_context(),
)
finally:
content.close()
finally:
UPLOAD_GUARD.release()
if attachment is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(202, attachment, request_id)
def _handle_get_attachment(self, attachment_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
if attachment is None:
raise ApiError(404, "not_found", "Attachment not found")
self._send_json(200, attachment, request_id)
def _handle_download_attachment(
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
):
actor = self._get_actor(request_id)
if not valid_uuid(attachment_id):
raise ApiError(400, "invalid_request", "Invalid attachment id")
variants = query.get("variant", ["original"])
if len(variants) != 1 or variants[0] not in ("original", "thumbnail", "crop", "optimized"):
raise ApiError(400, "invalid_request", "Invalid attachment variant")
variant = variants[0]
if variant != "original" and variant not in ("thumbnail", "crop", "optimized"):
raise ApiError(404, "not_found", "Attachment variant not found")
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"], variant)
if result is None:
raise ApiError(404, "not_found", "Attachment not found")
content_type, content = result
self._set_response_status(200)
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(content)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Cache-Control", "private, no-store")
self.end_headers()
self.wfile.write(content)
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if result is None:
raise ApiError(404, "not_found", "Chat not found")
items, next_cursor = result
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
def _send_json(
self,
status: int,
payload: Dict[str, Any],
request_id: str,
headers: Optional[Dict[str, str]] = None,
):
self._set_response_status(status)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
if headers:
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def _send_error(
self,
status: int,
code: str,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
+33
View File
@@ -0,0 +1,33 @@
import json
import os
import ssl
import pika
from .config import secret_value
QUEUE_NAME = "attachments.process.v1"
DEAD_QUEUE_NAME = "attachments.process.dead"
LEASE_NETWORK_BUDGET_SECONDS = 150
def broker_url() -> str:
return secret_value("BROKER_URL")
def connect() -> pika.BlockingConnection:
parameters = pika.URLParameters(broker_url())
context = ssl.create_default_context(cafile=os.environ["BROKER_CA_FILE"])
context.load_cert_chain(
os.environ["BROKER_CLIENT_CERT_FILE"],
os.environ["BROKER_CLIENT_KEY_FILE"],
)
parameters.ssl_options = pika.SSLOptions(context, parameters.host)
parameters.connection_attempts = 3
parameters.retry_delay = 1
parameters.socket_timeout = 5
parameters.blocked_connection_timeout = 10
return pika.BlockingConnection(parameters)
def encode_job(job: dict) -> bytes:
return json.dumps(job, sort_keys=True, separators=(",", ":")).encode("utf-8")
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
import logging
import os
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
def main():
logging.basicConfig(level="INFO")
store = Store()
store.open()
try:
store.migrate()
worker_password = Path(os.environ["WORKER_DB_PASSWORD_FILE"]).read_text(encoding="utf-8").strip()
store.configure_worker_role(worker_password)
finally:
store.close()
if __name__ == "__main__":
main()
+101
View File
@@ -0,0 +1,101 @@
import logging
import os
from datetime import datetime
from typing import BinaryIO, Iterator, Optional, Tuple, Union
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from .config import (
S3_ACCESS_KEY_ID,
S3_BUCKET,
S3_ENDPOINT,
S3_REGION,
S3_SECRET_ACCESS_KEY,
S3_STAGING_BUCKET,
S3_VARIANTS_BUCKET,
)
from .errors import ObjectMissingError
LOGGER = logging.getLogger("messenger.s3")
def bucket_for_key(key: str) -> str:
if key.startswith("staging/"):
return S3_STAGING_BUCKET
if key.startswith("original/"):
return S3_BUCKET
if key.startswith("variants/"):
return S3_VARIANTS_BUCKET
raise ValueError("Object key is outside configured storage namespaces")
class ObjectStore:
def __init__(self):
self.client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
verify=os.getenv("S3_CA_FILE", True),
config=Config(
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
client_cert=(
os.getenv("S3_CLIENT_CERT_FILE"),
os.getenv("S3_CLIENT_KEY_FILE"),
),
s3={"addressing_style": "path"},
),
)
def put(
self,
key: str,
content: Union[bytes, BinaryIO],
content_type: str,
size_bytes: Optional[int] = None,
) -> None:
if size_bytes is None:
size_bytes = len(content)
self.client.put_object(
Bucket=bucket_for_key(key),
Key=key,
Body=content,
ContentLength=size_bytes,
ContentType=content_type,
)
def get(self, key: str, max_bytes: int) -> bytes:
try:
response = self.client.get_object(Bucket=bucket_for_key(key), Key=key)
except ClientError as exc:
code = exc.response.get("Error", {}).get("Code")
if code in ("NoSuchKey", "404", "NotFound"):
raise ObjectMissingError from exc
raise
body = response["Body"]
try:
value = body.read(max_bytes + 1)
finally:
body.close()
if len(value) > max_bytes:
raise RuntimeError("Stored object exceeds configured safety limit")
return value
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=bucket_for_key(key), Key=key)
def ready(self) -> bool:
self.client.head_bucket(Bucket=S3_BUCKET)
return True
def objects(self) -> Iterator[Tuple[str, datetime]]:
for bucket in (S3_STAGING_BUCKET, S3_BUCKET, S3_VARIANTS_BUCKET):
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=bucket)
for page in pages:
for item in page.get("Contents", []):
yield item["Key"], item["LastModified"]
+4
View File
@@ -0,0 +1,4 @@
PIPELINE_QUALITY = {
1: {"jpeg": 85, "webp": 80},
2: {"jpeg": 80, "webp": 75},
}
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
import logging
import os
import time
import uuid
import pika
import psycopg
from opentelemetry import propagate
from opentelemetry.instrumentation.utils import suppress_instrumentation
from opentelemetry.trace import SpanKind
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect, encode_job
from .security import isoformat
from .store import Store
from .telemetry import initialize, tracer
LOGGER = logging.getLogger("messenger.publisher")
def publish_pending(store: Store, channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
with store.pool.connection() as conn:
with conn.transaction():
with suppress_instrumentation():
row = store.claim_outbox(conn)
if row is None:
return False
payload = {
"schema_version": 1,
"job_id": str(row["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"created_at": isoformat(row["created_at"]),
}
parent_context = propagate.extract(
{
key: row.get(key)
for key in ("traceparent", "tracestate")
if row.get(key) is not None
}
)
with tracer().start_as_current_span(
"attachment.publish",
context=parent_context,
kind=SpanKind.PRODUCER,
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "publish"},
):
carrier = {}
propagate.inject(carrier)
payload.update(carrier)
channel.basic_publish(
exchange="",
routing_key=QUEUE_NAME,
body=encode_job(payload),
properties=pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=str(row["id"]),
type="attachment.process.v1",
),
mandatory=True,
)
conn.execute(
"UPDATE attachment_outbox SET published_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
return True
def main() -> None:
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-publisher"))
store = Store()
store.open()
try:
while True:
connection = None
try:
connection = connect()
channel = connection.channel()
channel.confirm_delivery()
store.recover_expired_attachment_jobs(
int(os.getenv("JOB_MAX_ATTEMPTS", "5")),
int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) + LEASE_NETWORK_BUDGET_SECONDS,
)
if not publish_pending(store, channel):
time.sleep(1)
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("outbox publishing cycle failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
finally:
store.close()
if __name__ == "__main__":
main()
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import logging
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
LOGGER = logging.getLogger("messenger.reconcile")
def reconcile(
store: Store,
grace_seconds: int,
apply_changes: bool,
restore_dir: Optional[Path] = None,
):
started = time.monotonic()
metadata = store.reconciliation_snapshot()
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
found = set()
orphans = []
for key, last_modified in store.objects.objects():
found.add(key)
if key not in metadata and last_modified <= cutoff:
orphans.append(key)
missing = [
row for key, row in metadata.items()
if key not in found and row.get("reconcile_type", "original") == "original"
]
restore_candidates = []
if restore_dir is not None:
root = restore_dir.resolve(strict=True)
for row in missing:
source = restore_dir / str(row["id"])
try:
resolved = source.resolve(strict=True)
if (
resolved.parent != root
or not resolved.is_file()
or resolved.stat().st_size != row["size_bytes"]
):
continue
content = resolved.read_bytes()
except OSError:
continue
if (
len(content) == row["size_bytes"]
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
):
restore_candidates.append((row, content))
restored = 0
if apply_changes:
for key in orphans:
store.objects.delete(key)
restorable_ids = set()
for row, content in restore_candidates:
store.objects.put(row["object_key"], content, row["content_type"])
store.clear_missing_object(row["id"])
restorable_ids.add(row["id"])
restored += 1
for row in missing:
if row["id"] not in restorable_ids:
store.record_missing_object(row["id"])
for key, row in metadata.items():
if key in found and row.get("reconcile_type", "original") == "original":
store.clear_missing_object(row["id"])
unresolved_missing = len(missing) - restored
return {
"mode": "apply" if apply_changes else "dry-run",
"grace_seconds": grace_seconds,
"objects_seen": len(found),
"metadata_seen": len({str(row["id"]) for row in metadata.values()}),
"orphan_objects": len([key for key in found if key not in metadata]),
"stale_orphan_candidates": len(orphans),
"deleted_orphans": len(orphans) if apply_changes else 0,
"missing_objects": len(missing),
"unresolved_missing_objects": unresolved_missing,
"restorable_missing_objects": len(restore_candidates),
"restored_missing_objects": restored,
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
"duration_ms": round((time.monotonic() - started) * 1000, 3),
}
def main():
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
parser.add_argument("--grace-seconds", type=int, default=3600)
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
parser.add_argument(
"--restore-dir",
type=Path,
help="optional trusted backup directory with exact original bytes named by attachment UUID",
)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error("--grace-seconds must be at least 60")
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
store = Store()
store.open()
try:
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
print(json.dumps(result, sort_keys=True))
if result["unresolved_missing_objects"]:
LOGGER.error(
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
result["unresolved_missing_objects"],
)
finally:
store.close()
if __name__ == "__main__":
main()
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
import argparse
import json
import uuid
from .store import Store
def main() -> None:
parser = argparse.ArgumentParser(
description="Explicitly replay a failed attachment or start a newer pipeline version."
)
parser.add_argument("--attachment-id", required=True, type=uuid.UUID)
parser.add_argument(
"--pipeline-version",
required=True,
type=int,
help="pipeline version implemented by the deployed worker",
)
args = parser.parse_args()
if args.pipeline_version < 1:
parser.error("--pipeline-version must be positive")
store = Store()
store.open()
try:
result = store.replay_attachment_job(str(args.attachment_id), args.pipeline_version)
print(json.dumps(result, sort_keys=True))
finally:
store.close()
if __name__ == "__main__":
main()
+134
View File
@@ -0,0 +1,134 @@
import base64
import hashlib
import hmac
import os
import secrets
import threading
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from argon2.low_level import Type
_ARGON2_MEMORY_KIB = int(os.getenv("ARGON2_MEMORY_KIB", "19456"))
_ARGON2_ITERATIONS = int(os.getenv("ARGON2_ITERATIONS", "2"))
_ARGON2_PARALLELISM = int(os.getenv("ARGON2_PARALLELISM", "1"))
_PASSWORD_HASH_CONCURRENCY = int(os.getenv("PASSWORD_HASH_CONCURRENCY", "4"))
_PASSWORD_HASH_WAIT_SECONDS = float(os.getenv("PASSWORD_HASH_WAIT_SECONDS", "0.1"))
if (
_ARGON2_MEMORY_KIB < 19456
or _ARGON2_MEMORY_KIB > 65536
or not 2 <= _ARGON2_ITERATIONS <= 4
or not 1 <= _ARGON2_PARALLELISM <= 4
):
raise RuntimeError("Argon2id settings must be 19456..65536 KiB, t=2..4, and p=1..4")
if not 1 <= _PASSWORD_HASH_CONCURRENCY <= 4 or _PASSWORD_HASH_WAIT_SECONDS <= 0:
raise RuntimeError("Password hash concurrency must be 1..4 with a positive wait timeout")
_PASSWORD_HASHER = PasswordHasher(
time_cost=_ARGON2_ITERATIONS,
memory_cost=_ARGON2_MEMORY_KIB,
parallelism=_ARGON2_PARALLELISM,
type=Type.ID,
)
_PASSWORD_HASH_SLOTS = threading.BoundedSemaphore(_PASSWORD_HASH_CONCURRENCY)
class PasswordHashCapacityError(RuntimeError):
pass
@contextmanager
def _password_hash_slot():
if not _PASSWORD_HASH_SLOTS.acquire(timeout=_PASSWORD_HASH_WAIT_SECONDS):
raise PasswordHashCapacityError("password hashing capacity is temporarily exhausted")
try:
yield
finally:
_PASSWORD_HASH_SLOTS.release()
def isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def valid_uuid(value: str) -> bool:
try:
uuid.UUID(value)
return True
except (ValueError, TypeError, AttributeError):
return False
def hash_password(password: str) -> str:
with _password_hash_slot():
return _PASSWORD_HASHER.hash(password)
def _verify_password(password: str, stored: str) -> bool:
try:
if stored.startswith("$argon2id$"):
parts = stored.split("$")
if len(parts) != 6 or parts[2] != "v=19":
return False
parameters = dict(part.split("=", 1) for part in parts[3].split(","))
memory, iterations, parallelism = (
int(parameters["m"]),
int(parameters["t"]),
int(parameters["p"]),
)
if not (19456 <= memory <= 65536 and 2 <= iterations <= 4 and 1 <= parallelism <= 4):
return False
return _PASSWORD_HASHER.verify(stored, password)
# Existing lab6 accounts can log in once and be upgraded to Argon2id.
scheme, n, r, p, salt_text, hash_text = stored.split("$")
if scheme != "scrypt":
return False
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
actual = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=int(n),
r=int(r),
p=int(p),
dklen=len(expected),
maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(actual, expected)
except (
InvalidHashError,
VerificationError,
VerifyMismatchError,
ValueError,
TypeError,
UnicodeError,
IndexError,
KeyError,
):
return False
def verify_password(password: str, stored: str) -> bool:
with _password_hash_slot():
return _verify_password(password, stored)
def password_hash_needs_upgrade(stored: str) -> bool:
if not stored.startswith("$argon2id$"):
return True
try:
return _PASSWORD_HASHER.check_needs_rehash(stored)
except (InvalidHashError, VerificationError):
return True
def allowed_client_identity(cert: dict) -> bool:
return ("URI", "spiffe://mtusi2026lab7/load-balancer") in cert.get("subjectAltName", ())
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.app import main
else:
from .app import main
if __name__ == "__main__":
main()
+934
View File
@@ -0,0 +1,934 @@
import base64
import hashlib
import hmac
import json
import logging
import math
import os
import secrets
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
import redis
from psycopg import sql
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
IDEMPOTENCY_TTL_SECONDS,
SESSION_TTL_SECONDS,
secret_value,
)
from .errors import (
AttachmentNotReadyError,
CacheUnavailableError,
IdempotencyConflictError,
ObjectMissingError,
)
from .object_store import ObjectStore
from .pipeline_versions import PIPELINE_QUALITY
from .security import (
hash_password,
isoformat,
password_hash_needs_upgrade,
utc_now,
valid_uuid,
verify_password,
)
LOGGER = logging.getLogger("messenger.store")
_DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(32))
def database_url() -> str:
return secret_value("DATABASE_URL")
class Store:
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
self.objects = ObjectStore()
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
self.objects.ready()
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
def configure_worker_role(self, password: str) -> None:
with self.pool.connection() as conn:
with conn.transaction():
exists = conn.execute(
"SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker'"
).fetchone()
if exists:
statement = sql.SQL("ALTER ROLE messenger_worker LOGIN PASSWORD {}").format(
sql.Literal(password)
)
else:
statement = sql.SQL("CREATE ROLE messenger_worker LOGIN PASSWORD {}").format(
sql.Literal(password)
)
conn.execute(statement)
conn.execute("GRANT CONNECT ON DATABASE messenger TO messenger_worker")
conn.execute("GRANT USAGE ON SCHEMA public TO messenger_worker")
conn.execute(
"GRANT SELECT (id, job_id, pipeline_version, attempt, status, lease_until, "
"staging_key, object_key, content_type, size_bytes, sha256) ON attachments "
"TO messenger_worker"
)
conn.execute(
"GRANT UPDATE (status, lease_until, updated_at, variants, error_code, attempt) "
"ON attachments TO messenger_worker"
)
conn.execute(
"GRANT INSERT (id, job_id, attachment_id, pipeline_version, attempt, available_at, "
"traceparent, tracestate) ON attachment_outbox TO messenger_worker"
)
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
"attachment_ids": attachment_ids or [],
}
@staticmethod
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
variants = row.get("variants") or []
public_variants = [
{
name: variant[name]
for name in ("name", "content_type", "size_bytes", "width", "height", "sha256")
}
for variant in variants
]
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"].strip(),
"status": row.get("status", "queued"),
"variants": public_variants,
"error_code": row.get("error_code"),
"created_at": isoformat(row["created_at"]),
"updated_at": isoformat(row["updated_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
stored_hash = row["password_hash"] if row else _DUMMY_PASSWORD_HASH
if not verify_password(password, stored_hash) or not row:
return None
if password_hash_needs_upgrade(stored_hash):
upgraded_hash = hash_password(password)
with self.pool.connection() as conn:
conn.execute(
"UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s",
(upgraded_hash, row["id"], stored_hash),
)
return self.user(row)
def consume_login_attempt(self, username: str, source_ip: str) -> Optional[int]:
user_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
ip_key = hashlib.sha256(b"ip\0" + source_ip.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
def bump(key: bytes) -> Dict[str, Any]:
return conn.execute(
"""
INSERT INTO login_rate_limits (username_key, window_started_at, attempts)
VALUES (%s, clock_timestamp(), 1)
ON CONFLICT (username_key) DO UPDATE SET
attempts = CASE
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
THEN 1 ELSE login_rate_limits.attempts + 1
END,
window_started_at = CASE
WHEN login_rate_limits.window_started_at <= clock_timestamp() - interval '1 minute'
THEN clock_timestamp() ELSE login_rate_limits.window_started_at
END
RETURNING attempts,
GREATEST(1, ceil(extract(epoch FROM
(window_started_at + interval '1 minute' - clock_timestamp()))))::integer AS retry_after
""",
(key,),
).fetchone()
user_row = bump(user_key)
ip_row = bump(ip_key)
conn.execute(
"DELETE FROM login_rate_limits WHERE window_started_at < clock_timestamp() - interval '1 hour'"
)
retries = []
if user_row["attempts"] > 5:
retries.append(user_row["retry_after"])
if ip_row["attempts"] > 60:
retries.append(ip_row["retry_after"])
return max(retries) if retries else None
def clear_login_attempts(self, username: str) -> None:
username_key = hashlib.sha256(b"user\0" + username.lower().encode("utf-8")).digest()
with self.pool.connection() as conn:
conn.execute("DELETE FROM login_rate_limits WHERE username_key = %s", (username_key,))
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
attachment_ids: List[str],
idempotency_key: str,
canonical_body: str,
) -> Optional[Dict[str, Any]]:
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
if attachment_ids:
attachments = conn.execute(
"SELECT id, status FROM attachments "
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
(chat_id, attachment_ids),
).fetchall()
if len(attachments) != len(attachment_ids):
return None
if any(item["status"] == "failed" for item in attachments):
raise AttachmentNotReadyError
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
existing = conn.execute(
"SELECT request_hash, response, expires_at "
"FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
"FOR UPDATE",
(sender_id, chat_id, idempotency_key),
).fetchone()
if existing and existing["expires_at"] > utc_now():
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
raise IdempotencyConflictError
return existing["response"]
if existing:
conn.execute(
"DELETE FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
(sender_id, chat_id, idempotency_key),
)
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
for ordinal, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
"VALUES (%s, %s, %s)",
(row["id"], attachment_id, ordinal),
)
message = self.message(row, attachment_ids)
conn.execute(
"INSERT INTO idempotency_records "
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
(
sender_id,
chat_id,
idempotency_key,
request_hash,
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
),
)
return message
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
ids_by_message = {}
if rows:
links = conn.execute(
"SELECT message_id, attachment_id FROM message_attachments "
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
([row["id"] for row in rows],),
).fetchall()
for link in links:
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
with self.pool.connection() as conn:
return conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone() is not None
def create_attachment(
self,
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
content: BinaryIO,
size_bytes: int,
digest: str,
traceparent: Optional[str] = None,
tracestate: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
if not self.can_access_chat(chat_id, owner_id):
return None
attachment_id = uuid.uuid4()
job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}"
object_key = f"original/{attachment_id}"
content.seek(0)
self.objects.put(staging_key, content, content_type, size_bytes)
try:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO attachments "
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"object_key, staging_key, job_id) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, variants, error_code, created_at, updated_at",
(
attachment_id,
chat_id,
owner_id,
filename,
content_type,
size_bytes,
digest,
object_key,
staging_key,
job_id,
),
).fetchone()
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"VALUES (%s, %s, %s, 1, 1, %s, %s)",
(uuid.uuid4(), job_id, attachment_id, traceparent, tracestate),
)
return self.attachment(row)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"attachment staging compensation failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
"a.sha256, a.status, a.variants, a.error_code, a.created_at, a.updated_at "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
return self.attachment(row) if row else None
def download_attachment(
self, attachment_id: str, user_id: str, variant: str = "original"
) -> Optional[Tuple[str, bytes]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT a.object_key, a.staging_key, a.content_type, a.size_bytes, a.status, a.variants "
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = %s AND m.user_id = %s",
(attachment_id, user_id),
).fetchone()
if not row:
return None
if row["status"] != "ready":
raise AttachmentNotReadyError
if variant == "original":
key, content_type, size_bytes = row["object_key"], row["content_type"], row["size_bytes"]
else:
match = next(
(item for item in (row["variants"] or []) if item["name"] == variant),
None,
)
if match is None:
return None
key, content_type, size_bytes = match["object_key"], match["content_type"], match["size_bytes"]
return content_type, self.objects.get(key, size_bytes)
def claim_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int, lease_seconds: int
) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, lease_until, staging_key, "
"object_key, content_type, size_bytes, sha256 "
"FROM attachments WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
not row
or str(row["job_id"]) != job_id
or row["pipeline_version"] != pipeline_version
or row["status"] in ("ready", "failed")
or row["attempt"] != attempt
):
return None
if row["status"] == "processing" and row["lease_until"] is not None:
if row["lease_until"] > utc_now():
return None
return conn.execute(
"UPDATE attachments SET status = 'processing', "
"lease_until = clock_timestamp() + (%s * interval '1 second'), "
"updated_at = clock_timestamp() "
"WHERE id = %s RETURNING id, job_id, staging_key, object_key, content_type, "
"size_bytes, sha256, attempt, pipeline_version",
(lease_seconds, attachment_id),
).fetchone()
def complete_attachment_job(
self,
attachment_id: str,
job_id: str,
pipeline_version: int,
attempt: int,
variants: List[Dict[str, Any]],
) -> bool:
with self.pool.connection() as conn:
with conn.transaction():
updated = conn.execute(
"UPDATE attachments SET status = 'ready', variants = %s::jsonb, error_code = NULL, "
"lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s AND attempt = %s "
"AND status = 'processing' AND lease_until > clock_timestamp()",
(
json.dumps(variants, separators=(",", ":")),
attachment_id,
job_id,
pipeline_version,
attempt,
),
)
return updated.rowcount == 1
def fail_attachment_job(
self, attachment_id: str, job_id: str, pipeline_version: int, attempt: int,
max_attempts: int, error_code: str, retry_delay_seconds: Optional[float],
traceparent: Optional[str] = None, tracestate: Optional[str] = None,
) -> None:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"SELECT status, attempt FROM attachments "
"WHERE id = %s AND job_id = %s AND pipeline_version = %s FOR UPDATE",
(attachment_id, job_id, pipeline_version),
).fetchone()
if not row or row["status"] != "processing" or row["attempt"] != attempt:
return
if retry_delay_seconds is not None and attempt < max_attempts:
next_attempt = attempt + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, error_code, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"VALUES (%s, %s, %s, %s, %s, clock_timestamp() + (%s * interval '1 second'), %s, %s) "
"ON CONFLICT (job_id, attempt) DO NOTHING",
(
uuid.uuid4(), job_id, attachment_id, pipeline_version,
next_attempt, retry_delay_seconds, traceparent, tracestate,
),
)
else:
final_code = "retry_exhausted" if attempt >= max_attempts else error_code
conn.execute(
"UPDATE attachments SET status = 'failed', lease_until = NULL, error_code = %s, "
"updated_at = clock_timestamp() WHERE id = %s",
(final_code, attachment_id),
)
def replay_attachment_job(self, attachment_id: str, pipeline_version: int) -> Dict[str, Any]:
if pipeline_version not in PIPELINE_QUALITY:
raise ValueError("pipeline version is not implemented by this deployment")
with self.pool.connection() as conn:
current = conn.execute(
"SELECT id, job_id, pipeline_version, attempt, status, staging_key, object_key, "
"content_type, size_bytes, sha256 FROM attachments WHERE id = %s",
(attachment_id,),
).fetchone()
if current is None:
raise ValueError("attachment not found")
if pipeline_version < current["pipeline_version"]:
raise ValueError("pipeline version cannot move backwards")
if pipeline_version == current["pipeline_version"] and current["status"] != "failed":
raise ValueError("same-version replay is allowed only for failed attachments")
if current["status"] not in ("queued", "processing", "ready", "failed"):
raise ValueError("attachment cannot be replayed in its current state")
try:
content = self.objects.get(current["object_key"], current["size_bytes"])
except ObjectMissingError as object_error:
try:
content = self.objects.get(current["staging_key"], current["size_bytes"])
except ObjectMissingError:
raise object_error
if (
len(content) != current["size_bytes"]
or hashlib.sha256(content).hexdigest() != current["sha256"].strip()
):
raise ValueError("stored original does not match attachment metadata")
prior_job_id = str(current["job_id"])
next_job_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}/{next_job_id}"
self.objects.put(staging_key, content, current["content_type"], len(content))
try:
with self.pool.connection() as conn:
with conn.transaction():
locked = conn.execute(
"SELECT job_id, pipeline_version, attempt, status FROM attachments "
"WHERE id = %s FOR UPDATE",
(attachment_id,),
).fetchone()
if (
locked is None
or str(locked["job_id"]) != prior_job_id
or locked["pipeline_version"] != current["pipeline_version"]
or locked["attempt"] != current["attempt"]
or locked["status"] != current["status"]
):
raise RuntimeError("attachment changed while preparing replay")
conn.execute(
"UPDATE attachments SET job_id = %s, pipeline_version = %s, attempt = 1, "
"status = 'queued', staging_key = %s, variants = '[]'::jsonb, "
"error_code = NULL, lease_until = NULL, updated_at = clock_timestamp() "
"WHERE id = %s",
(next_job_id, pipeline_version, staging_key, attachment_id),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, 1)",
(uuid.uuid4(), next_job_id, attachment_id, pipeline_version),
)
conn.execute(
"INSERT INTO attachment_pipeline_events "
"(attachment_id, prior_job_id, job_id, prior_version, pipeline_version, action) "
"VALUES (%s, %s, %s, %s, %s, 'replay')",
(
attachment_id,
prior_job_id,
next_job_id,
current["pipeline_version"],
pipeline_version,
),
)
except Exception:
try:
self.objects.delete(staging_key)
except Exception as cleanup_error:
LOGGER.error(
"replay staging cleanup failed for %s (%s)",
attachment_id,
type(cleanup_error).__name__,
)
raise
return {
"attachment_id": str(attachment_id),
"previous_job_id": prior_job_id,
"job_id": str(next_job_id),
"pipeline_version": pipeline_version,
"attempt": 1,
"status": "queued",
}
def recover_expired_attachment_jobs(self, max_attempts: int, lease_seconds: int) -> int:
with self.pool.connection() as conn:
with conn.transaction():
rows = conn.execute(
"SELECT id, job_id, pipeline_version, attempt FROM attachments "
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
"ORDER BY lease_until FOR UPDATE SKIP LOCKED LIMIT 100"
).fetchall()
for row in rows:
if row["attempt"] >= max_attempts:
conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'retry_exhausted', "
"lease_until = NULL, updated_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)
continue
next_attempt = row["attempt"] + 1
conn.execute(
"UPDATE attachments SET status = 'queued', attempt = %s, lease_until = NULL, "
"error_code = 'worker_lease_expired', updated_at = clock_timestamp() WHERE id = %s",
(next_attempt, row["id"]),
)
conn.execute(
"INSERT INTO attachment_outbox "
"(id, job_id, attachment_id, pipeline_version, attempt) "
"VALUES (%s, %s, %s, %s, %s) ON CONFLICT (job_id, attempt) DO NOTHING",
(uuid.uuid4(), row["job_id"], row["id"], row["pipeline_version"], next_attempt),
)
return len(rows)
def claim_outbox(self, conn):
return conn.execute(
"SELECT o.id, o.job_id, o.attachment_id, o.pipeline_version, o.attempt, "
"o.traceparent, o.tracestate, "
"a.created_at FROM attachment_outbox o "
"JOIN attachments a ON a.id = o.attachment_id "
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
"ORDER BY o.created_at FOR UPDATE OF o SKIP LOCKED LIMIT 1"
).fetchone()
def queue_snapshot(self) -> Tuple[int, float]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, "
"COALESCE(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS oldest_age "
"FROM attachments WHERE status = 'queued'"
).fetchone()
return int(row["depth"]), float(row["oldest_age"])
def reconciliation_snapshot(self):
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT id, object_key, staging_key, status, variants, pipeline_version, created_at, "
"size_bytes, sha256, content_type FROM attachments"
).fetchall()
expected = {}
for row in rows:
if row["status"] == "ready":
expected[row["object_key"]] = {**row, "reconcile_type": "original"}
for variant in row["variants"] or []:
expected[variant["object_key"]] = {
**row,
"object_key": variant["object_key"],
"size_bytes": variant["size_bytes"],
"sha256": variant["sha256"],
"content_type": variant["content_type"],
"reconcile_type": "variant",
}
elif row["status"] in ("queued", "processing"):
expected[row["staging_key"]] = {**row, "reconcile_type": "staging"}
expected[row["object_key"]] = {**row, "reconcile_type": "inflight"}
for name, extension in (
("thumbnail", "jpg"),
("crop", "jpg"),
("optimized", "webp"),
):
key = f"variants/{row['id']}/v{row['pipeline_version']}/{name}.{extension}"
expected[key] = {**row, "object_key": key, "reconcile_type": "inflight"}
return expected
def record_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
"DO UPDATE SET last_seen_at = clock_timestamp()",
(attachment_id,),
)
def clear_missing_object(self, attachment_id: str) -> None:
with self.pool.connection() as conn:
conn.execute(
"DELETE FROM reconciliation_incidents "
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
(attachment_id,),
)
+300
View File
@@ -0,0 +1,300 @@
import contextvars
import json
import logging
import os
import platform
import socket
import threading
import time
from datetime import datetime, timezone
from typing import Optional
import psycopg
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_OFF, ALWAYS_ON, ParentBased, TraceIdRatioBased
from opentelemetry.instrumentation.utils import suppress_instrumentation
REQUEST_ID = contextvars.ContextVar("request_id", default="")
_lock = threading.Lock()
_runtime = {
"pool_used": 0,
"pool_waiting": 0,
"queue_depth": 0,
"queue_oldest_seconds": 0.0,
"worker_busy": 0,
"worker_capacity": 0,
"cpu": 0.0,
"memory": 0,
}
_provider = None
_meter = None
_service_name = "messenger"
_enabled = True
_request_counter = None
_request_duration = None
_job_counter = None
_job_duration = None
_time_to_ready = None
_queue_depth = None
_queue_oldest = None
_pool_used = None
_pool_waiting = None
_worker_busy = None
_worker_capacity = None
_process_cpu = None
_process_memory = None
class JsonLogFormatter(logging.Formatter):
converter = time.gmtime
def format(self, record):
span_context = trace.get_current_span().get_span_context()
document = {
"timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
"severity": record.levelname,
"service.name": _service_name,
"event": record.getMessage(),
"request_id": REQUEST_ID.get(),
}
if span_context.is_valid:
document["trace_id"] = format(span_context.trace_id, "032x")
document["span_id"] = format(span_context.span_id, "016x")
if record.exc_info:
document["exception.type"] = record.exc_info[0].__name__
return json.dumps(document, ensure_ascii=True, separators=(",", ":"))
def _sampler():
if os.getenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") == "always_off":
return ALWAYS_OFF
if os.getenv("OTEL_TRACES_SAMPLER", "") == "always_on":
return ALWAYS_ON
ratio = float(os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0"))
if not 0 <= ratio <= 1:
raise ValueError("OTEL_TRACES_SAMPLER_ARG must be between 0 and 1")
return ParentBased(TraceIdRatioBased(ratio))
def initialize(service_name: str) -> None:
global _enabled
global _service_name
global _provider, _meter
global _request_counter, _request_duration, _job_counter, _job_duration, _time_to_ready
global _queue_depth, _queue_oldest, _pool_used, _pool_waiting
global _worker_busy, _worker_capacity, _process_cpu, _process_memory
enabled = os.getenv("OTEL_SDK_DISABLED", "false").lower() != "true"
_enabled = enabled
_service_name = service_name
instance_id = os.getenv("INSTANCE_ID", socket.gethostname())
resource = Resource.create(
{
"service.name": service_name,
"service.version": os.getenv("SERVICE_VERSION", "lab6"),
"service.instance.id": instance_id,
"deployment.environment.name": os.getenv("APP_ENV", "lab"),
"host.name": socket.gethostname(),
"os.type": platform.system().lower(),
}
)
if enabled:
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
_provider = TracerProvider(resource=resource, sampler=_sampler())
_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint.rstrip("/") + "/v1/traces"),
max_queue_size=4096,
max_export_batch_size=512,
schedule_delay_millis=1000,
export_timeout_millis=10000,
)
)
trace.set_tracer_provider(_provider)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint.rstrip("/") + "/v1/metrics"),
export_interval_millis=10000,
export_timeout_millis=5000,
)
metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[reader]))
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint.rstrip("/") + "/v1/logs"),
max_queue_size=4096,
max_export_batch_size=512,
schedule_delay_millis=1000,
export_timeout_millis=10000,
)
)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otel_handler.setFormatter(JsonLogFormatter())
else:
trace.set_tracer_provider(TracerProvider(resource=resource, sampler=ALWAYS_OFF))
metrics.set_meter_provider(MeterProvider(resource=resource))
otel_handler = None
root = logging.getLogger()
root.setLevel(os.getenv("LOG_LEVEL", "INFO"))
formatter = JsonLogFormatter()
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
root.handlers[:] = [stream_handler]
if otel_handler is not None:
root.addHandler(otel_handler)
logging.getLogger("pika").setLevel(logging.WARNING)
if enabled:
try:
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
BotocoreInstrumentor().instrument()
PsycopgInstrumentor().instrument()
except ImportError as exc:
raise RuntimeError("OTel database and S3 instrumentation dependencies are required") from exc
_meter = metrics.get_meter("messenger.telemetry")
_request_counter = _meter.create_counter("messenger.http.requests", unit="{request}")
_request_duration = _meter.create_histogram("messenger.http.duration", unit="s")
_job_counter = _meter.create_counter("messenger.jobs.processed", unit="{job}")
_job_duration = _meter.create_histogram("messenger.jobs.duration", unit="s")
_time_to_ready = _meter.create_histogram("messenger.attachments.time_to_ready", unit="s")
def observe(name, attributes=None):
def callback(_options):
with _lock:
return [metrics.Observation(_runtime[name], attributes=attributes or {})]
return callback
_queue_depth = _meter.create_observable_gauge(
"messenger.queue.depth",
callbacks=[observe("queue_depth", {"queue.name": "attachments.process.v1"})],
unit="{job}",
)
_queue_oldest = _meter.create_observable_gauge(
"messenger.queue.oldest_age",
callbacks=[observe("queue_oldest_seconds", {"queue.name": "attachments.process.v1"})],
unit="s",
)
_pool_used = _meter.create_observable_gauge(
"messenger.db.pool.connections.used",
callbacks=[observe("pool_used", {"pool.name": "primary"})],
unit="{connection}",
)
_pool_waiting = _meter.create_observable_gauge(
"messenger.db.pool.requests.waiting",
callbacks=[observe("pool_waiting", {"pool.name": "primary"})],
unit="{request}",
)
_worker_busy = _meter.create_observable_gauge(
"messenger.worker.busy", callbacks=[observe("worker_busy")], unit="{worker}"
)
_worker_capacity = _meter.create_observable_gauge(
"messenger.worker.capacity", callbacks=[observe("worker_capacity")], unit="{worker}"
)
_process_cpu = _meter.create_observable_gauge(
"messenger.process.cpu.utilization", callbacks=[observe("cpu")], unit="1"
)
_process_memory = _meter.create_observable_gauge(
"messenger.process.memory.usage", callbacks=[observe("memory")], unit="By"
)
def tracer():
return trace.get_tracer("messenger.application")
def record_http_request(method: str, route: str, status: int, duration: float) -> None:
attributes = {
"http.request.method": method,
"http.route": route,
"status_class": f"{status // 100}xx",
}
if _request_counter is not None:
_request_counter.add(1, attributes)
_request_duration.record(duration, attributes)
if _enabled and not route.startswith("/health/"):
logging.getLogger("messenger.http").info(
"http request completed method=%s route=%s status_class=%s",
method,
route,
f"{status // 100}xx",
)
def record_job(outcome: str, duration: float, time_to_ready: Optional[float] = None) -> None:
attributes = {"job.type": "attachment.process.v1", "outcome": outcome}
if _job_counter is not None:
_job_counter.add(1, attributes)
_job_duration.record(duration, attributes)
if time_to_ready is not None:
_time_to_ready.record(time_to_ready, {"outcome": outcome})
def update_runtime(**values) -> None:
with _lock:
_runtime.update(values)
def start_runtime_metrics(
store, worker_capacity: int = 0, include_queue_snapshot: bool = True
) -> Optional[threading.Thread]:
if not _enabled:
return None
def refresh():
try:
import psutil
process = psutil.Process()
process.cpu_percent(interval=None)
while True:
try:
stats = store.pool.get_stats()
if include_queue_snapshot:
with suppress_instrumentation():
queue_depth, oldest_age = store.queue_snapshot()
cpu = process.cpu_percent(interval=1) / 100.0
memory = process.memory_info().rss
runtime = dict(
pool_used=max(0, stats.get("pool_size", 0) - stats.get("pool_available", 0)),
pool_waiting=stats.get("requests_waiting", 0),
worker_capacity=worker_capacity,
cpu=cpu,
memory=memory,
)
if include_queue_snapshot:
runtime.update(queue_depth=queue_depth, queue_oldest_seconds=oldest_age)
update_runtime(**runtime)
except (psycopg.Error, OSError, RuntimeError, ValueError):
logging.getLogger("messenger.telemetry").exception("runtime metric sample failed")
time.sleep(9)
except (OSError, RuntimeError, ValueError):
logging.getLogger("messenger.telemetry").exception("runtime metric sampler stopped")
raise
thread = threading.Thread(target=refresh, name="runtime-metrics", daemon=True)
thread.start()
return thread
def worker_started() -> None:
with _lock:
_runtime["worker_busy"] += 1
def worker_stopped() -> None:
with _lock:
_runtime["worker_busy"] = max(0, _runtime["worker_busy"] - 1)
+383
View File
@@ -0,0 +1,383 @@
#!/usr/bin/env python3
import hashlib
import io
import json
import logging
import multiprocessing
import os
import random
import time
import uuid
import warnings
import zlib
from datetime import datetime, timezone
from typing import Callable, Optional
import pika
import psycopg
from opentelemetry import propagate
from opentelemetry.trace import SpanKind, Status, StatusCode
from botocore.exceptions import BotoCoreError, ClientError
from PIL import Image, ImageOps, UnidentifiedImageError
from .config import MAX_IMAGE_PIXELS
from .jobs import LEASE_NETWORK_BUDGET_SECONDS, QUEUE_NAME, connect
from .pipeline_versions import PIPELINE_QUALITY
from .store import Store
from .telemetry import (
initialize,
record_job,
start_runtime_metrics,
tracer,
worker_started,
worker_stopped,
)
LOGGER = logging.getLogger("messenger.worker")
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "1"))
class InvalidAttachment(Exception):
pass
class UnsupportedPipeline(InvalidAttachment):
pass
def _rgb_image(content: bytes) -> Image.Image:
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(content)) as source:
if source.format not in ("PNG", "JPEG"):
raise InvalidAttachment("unsupported_image")
width, height = source.size
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
raise InvalidAttachment("image_too_large")
source.load()
image = ImageOps.exif_transpose(source)
try:
if "A" in image.getbands() or "transparency" in image.info:
with image.convert("RGBA") as rgba:
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return image.convert("RGB")
finally:
if image is not source:
image.close()
except InvalidAttachment:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning, OSError, SyntaxError,
UnidentifiedImageError, ValueError, zlib.error) as exc:
raise InvalidAttachment("invalid_image") from exc
def _encode(image: Image.Image, image_format: str, **options) -> bytes:
output = io.BytesIO()
image.save(output, format=image_format, **options)
return output.getvalue()
def build_variants(attachment_id: str, content: bytes, pipeline_version: int = 1):
quality = PIPELINE_QUALITY.get(pipeline_version)
if quality is None:
raise UnsupportedPipeline("unsupported_pipeline_version")
image = _rgb_image(content)
try:
variants = []
thumbnail = image.copy()
try:
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumbnail_bytes = _encode(thumbnail, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("thumbnail", "image/jpeg", thumbnail.width, thumbnail.height, thumbnail_bytes))
finally:
thumbnail.close()
crop = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5))
try:
crop_bytes = _encode(crop, "JPEG", quality=quality["jpeg"], optimize=True)
variants.append(("crop", "image/jpeg", 128, 128, crop_bytes))
finally:
crop.close()
optimized = image.copy()
try:
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
optimized_bytes = _encode(optimized, "WEBP", quality=quality["webp"], method=6)
variants.append(("optimized", "image/webp", optimized.width, optimized.height, optimized_bytes))
finally:
optimized.close()
result = []
for name, content_type, width, height, data in variants:
extension = "webp" if content_type == "image/webp" else "jpg"
key = f"variants/{attachment_id}/v{pipeline_version}/{name}.{extension}"
result.append(
{
"name": name,
"content_type": content_type,
"size_bytes": len(data),
"width": width,
"height": height,
"sha256": hashlib.sha256(data).hexdigest(),
"object_key": key,
"content": data,
}
)
return result
finally:
image.close()
class ProcessingTimeout(TimeoutError):
pass
def _variant_process(connection, attachment_id, content, pipeline_version, builder):
try:
connection.send(("ok", builder(attachment_id, content, pipeline_version)))
except InvalidAttachment as exc:
connection.send(("invalid", str(exc)))
except Exception as exc:
connection.send(("error", type(exc).__name__))
finally:
connection.close()
def _build_variants_bounded(
attachment_id: str,
content: bytes,
pipeline_version: int,
timeout_seconds: int,
builder=build_variants,
):
if timeout_seconds < 1:
raise ValueError("timeout_seconds must be positive")
context = multiprocessing.get_context("spawn")
receiver, sender = context.Pipe(duplex=False)
process = context.Process(
target=_variant_process,
args=(sender, attachment_id, content, pipeline_version, builder),
)
process.start()
sender.close()
try:
if not receiver.poll(timeout_seconds):
process.terminate()
process.join(timeout=2)
if process.is_alive():
process.kill()
process.join()
raise ProcessingTimeout("image processing exceeded its time budget")
try:
result, value = receiver.recv()
except EOFError as exc:
raise RuntimeError("image processing process exited without a result") from exc
process.join(timeout=2)
if process.is_alive():
process.terminate()
process.join()
if result == "invalid":
raise InvalidAttachment(value)
if result == "error":
raise RuntimeError(f"image processing failed in subprocess ({value})")
return value
finally:
receiver.close()
if process.is_alive():
process.terminate()
process.join()
def _job(payload: bytes):
value = json.loads(payload)
required = {
"schema_version", "job_id", "type", "attachment_id",
"pipeline_version", "attempt", "created_at",
}
if not isinstance(value, dict) or set(value).difference(required | {"traceparent", "tracestate"}):
raise ValueError("invalid job fields")
if not required.issubset(value) or value["schema_version"] != 1 or value["type"] != "attachment.process.v1":
raise ValueError("invalid job schema")
uuid.UUID(value["job_id"])
uuid.UUID(value["attachment_id"])
if not isinstance(value["pipeline_version"], int) or value["pipeline_version"] < 1:
raise ValueError("invalid pipeline version")
if not isinstance(value["attempt"], int) or value["attempt"] < 1:
raise ValueError("invalid job attempt")
return value
def _process(
store: Store,
job: dict,
fault_injector: Optional[Callable[[str], None]] = None,
) -> str:
def inject(point: str) -> None:
if fault_injector is not None:
fault_injector(point)
attachment_id = job["attachment_id"]
row = store.claim_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
JOB_TIMEOUT_SECONDS + LEASE_NETWORK_BUDGET_SECONDS,
)
if row is None:
LOGGER.info("job is duplicate, stale, terminal, or already leased: %s", job["job_id"])
return "skipped"
try:
inject("after_claim")
with tracer().start_as_current_span("attachment.read_staging"):
content = store.objects.get(row["staging_key"], row["size_bytes"])
if hashlib.sha256(content).hexdigest() != row["sha256"].strip():
raise InvalidAttachment("checksum_mismatch")
variants = _build_variants_bounded(
attachment_id,
content,
job["pipeline_version"],
JOB_TIMEOUT_SECONDS,
)
inject("before_original_put")
with tracer().start_as_current_span("attachment.write_original"):
store.objects.put(row["object_key"], content, row["content_type"], len(content))
inject("after_original_put")
persisted = []
for variant in variants:
data = variant.pop("content")
inject(f"before_variant_put:{variant['name']}")
with tracer().start_as_current_span("attachment.write_variant"):
store.objects.put(variant["object_key"], data, variant["content_type"], len(data))
inject(f"after_variant_put:{variant['name']}")
persisted.append(variant)
inject("before_commit")
with tracer().start_as_current_span("attachment.commit_ready"):
completed = store.complete_attachment_job(
attachment_id,
job["job_id"],
job["pipeline_version"],
job["attempt"],
persisted,
)
if not completed:
LOGGER.warning("job lease was superseded before completion: %s", job["job_id"])
return "skipped"
inject("after_commit")
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("completed attachment staging cleanup failed: %s", attachment_id)
LOGGER.info("attachment processing completed: %s", attachment_id)
return "success"
except InvalidAttachment as exc:
carrier = {}
propagate.inject(carrier)
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, str(exc), None, carrier.get("traceparent"), carrier.get("tracestate"),
)
try:
store.objects.delete(row["staging_key"])
except (BotoCoreError, ClientError, OSError):
LOGGER.exception("failed attachment staging cleanup failed: %s", attachment_id)
LOGGER.warning("attachment permanently failed: %s (%s)", attachment_id, exc)
return "failed"
except Exception as exc:
delay = min(60.0, 2 ** max(0, job["attempt"] - 1)) * random.uniform(0.75, 1.25)
carrier = {}
propagate.inject(carrier)
store.fail_attachment_job(
attachment_id, job["job_id"], job["pipeline_version"], job["attempt"],
MAX_ATTEMPTS, "processing_failed", delay, carrier.get("traceparent"), carrier.get("tracestate"),
)
LOGGER.exception("attachment processing attempt failed: %s (%s)", attachment_id, type(exc).__name__)
return "failed"
def handle_delivery(
store: Store,
channel: pika.adapters.blocking_connection.BlockingChannel,
method,
body: bytes,
fault_injector: Optional[Callable[[str], None]] = None,
) -> None:
try:
job = _job(body)
except (ValueError, TypeError, json.JSONDecodeError):
LOGGER.exception("invalid broker job rejected")
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
return
parent_context = propagate.extract(
{key: job[key] for key in ("traceparent", "tracestate") if key in job}
)
started = time.perf_counter()
created_at = datetime.fromisoformat(job["created_at"].replace("Z", "+00:00"))
with tracer().start_as_current_span(
"attachment.process",
context=parent_context,
kind=SpanKind.CONSUMER,
attributes={"messaging.destination.name": QUEUE_NAME, "messaging.operation.type": "process"},
) as span:
worker_started()
try:
outcome = _process(store, job, fault_injector)
if outcome == "failed":
span.set_status(Status(StatusCode.ERROR))
if outcome != "skipped":
elapsed = time.perf_counter() - started
ready_duration = (
max(0.0, datetime.now(timezone.utc).timestamp() - created_at.timestamp())
if outcome == "success"
else None
)
record_job(outcome, elapsed, ready_duration)
finally:
worker_stopped()
if fault_injector is not None:
fault_injector("before_ack")
channel.basic_ack(delivery_tag=method.delivery_tag)
def main() -> None:
if MAX_ATTEMPTS < 1 or JOB_TIMEOUT_SECONDS < 1 or WORKER_CONCURRENCY < 1:
raise RuntimeError("Worker limits must be positive")
initialize(os.getenv("OTEL_SERVICE_NAME", "messenger-worker"))
store = Store()
store.open()
start_runtime_metrics(store, WORKER_CONCURRENCY, include_queue_snapshot=False)
connection = None
try:
while True:
try:
connection = connect()
channel = connection.channel()
channel.basic_qos(prefetch_count=WORKER_CONCURRENCY)
def on_message(ch, method, properties, body):
handle_delivery(store, ch, method, body)
channel.basic_consume(queue=QUEUE_NAME, on_message_callback=on_message, auto_ack=False)
LOGGER.info("worker consuming durable attachment queue")
channel.start_consuming()
except (OSError, pika.exceptions.AMQPError, psycopg.Error):
LOGGER.exception("worker connection failed")
time.sleep(2)
finally:
if connection is not None and connection.is_open:
connection.close()
connection = None
finally:
store.close()
if __name__ == "__main__":
main()
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
import uuid
from pathlib import Path
import sys
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=202):
return client.call(
"POST",
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
expected,
actor=actor,
raw=data,
headers={"Content-Type": content_type},
schema="Attachment" if expected in (201, 202) else None,
)
def main():
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
other_chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
schema="Chat",
)["json"]
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
print("Checking accepted fixture upload.", flush=True)
attachment = upload(client, alice, chat["id"], image)["json"]
print("Checking damaged and spoofed image rejection.", flush=True)
upload(client, alice, chat["id"], b"not an image", expected=415)
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
print("Checking byte and pixel limits.", flush=True)
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
import struct
import zlib
def chunk(kind, payload):
return (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
)
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
bad_idat[-3] ^= 0xFF
damaged_idat = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", bytes(bad_idat))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], damaged_idat, expected=415)
too_many_pixels = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b""))
+ chunk(b"IEND", b"")
)
upload(client, alice, chat["id"], too_many_pixels, expected=413)
print("Checking filename and cross-chat authorization.", flush=True)
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
upload(client, eve, chat["id"], image, expected=404)
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
client.call(
"POST",
"/api/v1/chats/" + other_chat["id"] + "/messages",
404,
actor=alice,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": "", "attachment_ids": [attachment["id"]]},
)
client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
400,
actor=bob,
headers={"Idempotency-Key": uuid.uuid4().hex},
body={"text": ""},
)
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
if __name__ == "__main__":
run(main)
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Prove a concurrent idempotency-key race through the load balancer."""
from concurrent.futures import ThreadPoolExecutor
import threading
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def main():
args = arguments("Concurrent shared-idempotency integration check")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = "race-" + uuid.uuid4().hex
text = "concurrent-" + uuid.uuid4().hex
count = 12
barrier = threading.Barrier(count)
def send_once(_):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=10)
return worker.message(alice, chat, text, key=key)
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(send_once, range(count)))
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
messages = [response["json"] for response in responses]
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]
matches = [message for message in page["items"] if message["text"] == text]
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
if __name__ == "__main__":
run(main)
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Verify an upload committed while the publisher is stopped is processed after restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify committed attachment outbox intent survives publisher downtime")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
compose("stop", "publisher")
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted into queued state")
finally:
compose("start", "publisher")
ready = client.wait_ready(alice, attachment)
stored_original = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
thumbnail = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
variant = next(item for item in ready["variants"] if item["name"] == "thumbnail")
check(stored_original == original, "recovered job changed original bytes")
check(hashlib.sha256(thumbnail).hexdigest() == variant["sha256"], "recovered thumbnail checksum mismatch")
print(
json.dumps(
{
"result": "PASS",
"accepted_status_before_publisher_restart": attachment["status"],
"final_status": ready["status"],
"attachment_id": attachment["id"],
"original_sha256": hashlib.sha256(stored_original).hexdigest(),
"thumbnail_sha256": variant["sha256"],
"recovered_without_manual_republish": True,
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+485
View File
@@ -0,0 +1,485 @@
#!/usr/bin/env python3
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
import hashlib
import json
import os
from pathlib import Path
import sys
import time
import uuid
from datetime import datetime, timezone
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import pika
from src.publisher import publish_pending
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
JOB_ID = "00000000-0000-4000-8000-000000000002"
class SimulatedCrash(BaseException):
pass
class FakeObjects:
def __init__(self, content):
self.data = {"staging/" + ATTACHMENT_ID: content}
self.puts = []
def get(self, key, max_bytes):
return self.data[key]
def put(self, key, content, content_type, size_bytes=None):
value = content if isinstance(content, bytes) else content.read()
assert len(value) == size_bytes
self.data[key] = value
self.puts.append((key, hashlib.sha256(value).hexdigest()))
def delete(self, key):
self.data.pop(key, None)
class FakeStore:
def __init__(self, content):
self.objects = FakeObjects(content)
self.row = {
"id": ATTACHMENT_ID,
"job_id": JOB_ID,
"pipeline_version": 1,
"attempt": 1,
"status": "queued",
"lease_until": 0.0,
"staging_key": "staging/" + ATTACHMENT_ID,
"object_key": "original/" + ATTACHMENT_ID,
"content_type": "image/png",
"size_bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
"variants": [],
}
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] in ("ready", "failed")
):
return None
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
return None
row["status"] = "processing"
row["lease_until"] = time.monotonic() + lease_seconds
return dict(row)
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] != "processing"
or row["lease_until"] <= time.monotonic()
):
return False
row["status"] = "ready"
row["lease_until"] = 0.0
row["variants"] = variants
return True
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
row = self.row
if (
row["id"] == attachment_id
and row["job_id"] == job_id
and row["pipeline_version"] == version
and row["attempt"] == attempt
and row["status"] == "processing"
):
row["status"] = "failed" if delay is None else "queued"
row["lease_until"] = 0.0
row["error_code"] = error_code
if delay is not None:
row["attempt"] += 1
def recover_expired(self):
if self.row["status"] == "processing":
self.row["attempt"] += 1
self.row["status"] = "queued"
self.row["lease_until"] = 0.0
def replay(self, version):
row = self.row
previous_job = row["job_id"]
row["job_id"] = str(uuid.uuid4())
row["pipeline_version"] = version
row["attempt"] = 1
row["status"] = "queued"
row["lease_until"] = 0.0
row["variants"] = []
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
self.objects.put(
row["staging_key"],
self.objects.data[row["object_key"]],
row["content_type"],
row["size_bytes"],
)
return {
"schema_version": 1,
"job_id": row["job_id"],
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
"previous_job_id": previous_job,
}
class FakeChannel:
def __init__(self):
self.acked = []
self.nacked = []
def basic_ack(self, delivery_tag):
self.acked.append(delivery_tag)
def basic_nack(self, delivery_tag, requeue):
self.nacked.append((delivery_tag, requeue))
class FakeMethod:
def __init__(self, delivery_tag):
self.delivery_tag = delivery_tag
def job_body(job_id=JOB_ID, version=1, attempt=1):
return json.dumps(
{
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": attempt,
"created_at": "2026-01-01T00:00:00Z",
}
).encode()
def crash_at(expected):
def inject(point):
if point == expected:
raise SimulatedCrash(point)
return inject
def slow_builder(attachment_id, content, pipeline_version):
time.sleep(5)
def test_duplicate_and_ack_loss(content):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(1), job_body(),
fault_injector=crash_at("before_ack"),
)
raise AssertionError("fault injection did not crash before ack")
except SimulatedCrash:
pass
assert store.row["status"] == "ready"
first_write_count = len(store.objects.puts)
assert channel.acked == []
handle_delivery(store, channel, FakeMethod(2), job_body())
assert channel.acked == [2]
assert len(store.objects.puts) == first_write_count
assert store.row["status"] == "ready"
return {
"scenario": "duplicate_delivery_after_commit_before_ack",
"deliveries": 2,
"effective_object_writes": first_write_count,
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
}
def test_crash_before_and_after_side_effect(content, crash_point):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(3), job_body(),
fault_injector=crash_at(crash_point),
)
raise AssertionError(f"fault injection did not crash at {crash_point}")
except SimulatedCrash:
pass
assert channel.acked == []
store.recover_expired()
retry = job_body(attempt=2)
handle_delivery(store, channel, FakeMethod(4), retry)
assert store.row["status"] == "ready"
assert store.row["attempt"] == 2
assert channel.acked == [4]
assert store.objects.data["original/" + ATTACHMENT_ID] == content
return {
"scenario": "crash_" + crash_point,
"attempts": store.row["attempt"],
"terminal_status": store.row["status"],
}
def test_reordered_attempts(content):
store = FakeStore(content)
store.row["attempt"] = 2
channel = FakeChannel()
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
writes_after_current = len(store.objects.puts)
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
assert store.row["status"] == "ready"
assert len(store.objects.puts) == writes_after_current
assert channel.acked == [5, 6]
return {
"scenario": "out_of_order_attempt_delivery",
"current_attempt": store.row["attempt"],
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
}
def test_stale_pipeline_cannot_publish_newer_result(content):
store = FakeStore(content)
channel = FakeChannel()
replayed = []
def supersede_after_first_variant(point):
if point == "after_variant_put:thumbnail" and not replayed:
replayed.append(store.replay(2))
handle_delivery(
store, channel, FakeMethod(7), job_body(),
fault_injector=supersede_after_first_variant,
)
assert store.row["status"] == "queued"
assert store.row["pipeline_version"] == 2
assert channel.acked == [7]
current_job = replayed[0]
handle_delivery(
store,
channel,
FakeMethod(8),
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
)
assert store.row["status"] == "ready"
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
assert channel.acked == [7, 8]
return {
"scenario": "stale_pipeline_after_newer_version",
"current_pipeline_version": store.row["pipeline_version"],
"published_variant_versions": sorted(
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
),
}
class FakeTransaction:
def __init__(self, store):
self.store = store
def __enter__(self):
self.published = False
self.store.current_transaction = self
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None and self.published:
self.store.outbox_published = True
self.store.current_transaction = None
return False
class FakeConnection:
def __init__(self, store):
self.store = store
def transaction(self):
return FakeTransaction(self.store)
def execute(self, query, params):
assert "UPDATE attachment_outbox" in query
assert params == ("outbox-row",)
self.store.current_transaction.published = True
class FakePool:
def __init__(self, store):
self.store = store
def connection(self):
connection = FakeConnection(self.store)
class ConnectionContext:
def __enter__(self_inner):
return connection
def __exit__(self_inner, exc_type, exc_value, traceback):
return False
return ConnectionContext()
class FakeOutboxStore:
def __init__(self):
self.pool = FakePool(self)
self.outbox_published = False
self.current_transaction = None
def claim_outbox(self, connection):
if self.outbox_published:
return None
return {
"id": "outbox-row",
"job_id": JOB_ID,
"attachment_id": ATTACHMENT_ID,
"pipeline_version": 1,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
}
class AmbiguousConfirmChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
self.messages.append(message)
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
class FailBeforePublishChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
self.messages.append(message)
def test_committed_outbox_survives_publisher_crash():
store = FakeOutboxStore()
channel = FailBeforePublishChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher crash was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert channel.messages == []
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 1
return {
"scenario": "publisher_crash_before_publish",
"published_after_restart": len(channel.messages),
"outbox_recovered_after_restart": store.outbox_published,
}
def test_outbox_confirm_gap_republishes_same_job():
store = FakeOutboxStore()
channel = AmbiguousConfirmChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher confirm fault was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 2
first, second = channel.messages
assert first["body"] == second["body"]
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
return {
"scenario": "publish_confirm_lost_after_broker_accept",
"deliveries": len(channel.messages),
"same_message_id": True,
}
def test_image_processing_has_a_hard_timeout():
started = time.monotonic()
try:
_build_variants_bounded(
ATTACHMENT_ID,
b"bounded timeout test",
1,
timeout_seconds=1,
builder=slow_builder,
)
raise AssertionError("slow transform exceeded its time budget without termination")
except ProcessingTimeout:
pass
elapsed = time.monotonic() - started
assert elapsed < 4
return {
"scenario": "transform_timeout_terminated",
"configured_timeout_seconds": 1,
"observed_seconds": round(elapsed, 3),
}
def main():
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
results = [
test_duplicate_and_ack_loss(content),
test_crash_before_and_after_side_effect(content, "after_claim"),
test_crash_before_and_after_side_effect(content, "after_original_put"),
test_reordered_attempts(content),
test_stale_pipeline_cannot_publish_newer_result(content),
test_committed_outbox_survives_publisher_crash(),
test_outbox_confirm_gap_republishes_same_job(),
test_image_processing_has_a_hard_timeout(),
]
report = {
"result": "PASS",
"track": "2",
"fixture": "tests/fixtures/sample.png",
"scenarios": results,
"invariants": {
"confirmed_work_eventually_ready": True,
"duplicate_effects_create_no_second_variant_set": True,
"stale_attempt_writes_no_effects": True,
"stale_pipeline_cannot_publish_current_metadata": True,
"ambiguous_confirm_republishes_stable_job": True,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
},
}
evidence = ROOT / "evidence" / "pipeline-faults.json"
evidence.parent.mkdir(exist_ok=True)
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(report, sort_keys=True))
print(f"Evidence written to {evidence.relative_to(ROOT)}")
if __name__ == "__main__":
main()
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Verify explicit replay changes the deployed pipeline while preserving the original."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
MARK_FAILED_CODE = """
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
updated = conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'injected_test_failure', "
"variants = '[]'::jsonb, lease_until = NULL WHERE id = %s AND status = 'ready'",
(sys.argv[1],),
)
if updated.rowcount != 1:
raise RuntimeError("test attachment was not ready for failed replay injection")
finally:
store.close()
"""
AUDIT_CODE = """
import json
import sys
from src.store import Store
store = Store()
store.open()
try:
with store.pool.connection() as conn:
rows = conn.execute(
"SELECT prior_version, pipeline_version, action FROM attachment_pipeline_events "
"WHERE attachment_id = %s ORDER BY id",
(sys.argv[1],),
).fetchall()
print(json.dumps(rows, sort_keys=True))
finally:
store.close()
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def replay(container_id, attachment_id, version):
result = docker_exec(
container_id,
"python3",
"-m",
"src.replay_attachment",
"--attachment-id",
attachment_id,
"--pipeline-version",
str(version),
)
return json.loads(result.stdout)
def main():
args = arguments("Verify versioned attachment replay against the running Compose stack")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready_v1 = client.wait_ready(alice, attachment)
v1_thumbnail = next(item for item in ready_v1["variants"] if item["name"] == "thumbnail")
v1_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
check(hashlib.sha256(v1_bytes).hexdigest() == v1_thumbnail["sha256"], "v1 thumbnail checksum mismatch")
containers = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
).stdout.split()
check(bool(containers), "no running API container found")
replay_v2 = replay(containers[0], attachment["id"], 2)
check(replay_v2["status"] == "queued", "replay was not queued")
check(replay_v2["pipeline_version"] == 2, "replay did not select pipeline v2")
check(replay_v2["job_id"] != replay_v2["previous_job_id"], "replay reused the previous logical job id")
ready_v2 = client.wait_ready(alice, ready_v1)
v2_thumbnail = next(item for item in ready_v2["variants"] if item["name"] == "thumbnail")
v2_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail",
actor=alice,
)["bytes"]
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=alice,
)["bytes"]
check(v2_thumbnail["sha256"] != v1_thumbnail["sha256"], "pipeline v2 did not change thumbnail encoding")
check(hashlib.sha256(v2_bytes).hexdigest() == v2_thumbnail["sha256"], "v2 thumbnail checksum mismatch")
check(original_bytes == original, "pipeline replay changed the original file")
docker_exec(
containers[0],
"python3",
"-c",
MARK_FAILED_CODE,
attachment["id"],
)
failed_replay = replay(containers[0], attachment["id"], 2)
check(failed_replay["status"] == "queued", "failed attachment replay was not queued")
check(failed_replay["pipeline_version"] == 2, "failed replay changed pipeline version")
check(failed_replay["job_id"] != failed_replay["previous_job_id"], "failed replay reused a job id")
ready_replayed = client.wait_ready(alice, ready_v2)
audit = json.loads(
docker_exec(
containers[0],
"python3",
"-c",
AUDIT_CODE,
attachment["id"],
).stdout
)
check(
[(row["prior_version"], row["pipeline_version"], row["action"]) for row in audit]
== [(1, 2, "replay"), (2, 2, "replay")],
"pipeline replay audit trail is incomplete",
)
check(ready_replayed["status"] == "ready", "failed attachment did not recover")
print(
json.dumps(
{
"result": "PASS",
"previous_job_id": replay_v2["previous_job_id"],
"job_id": replay_v2["job_id"],
"pipeline_version": replay_v2["pipeline_version"],
"failed_replay_job_id": failed_replay["job_id"],
"audited_replays": len(audit),
"v1_thumbnail_sha256": v1_thumbnail["sha256"],
"v2_thumbnail_sha256": v2_thumbnail["sha256"],
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
import os
import hashlib
import sys
import tempfile
from pathlib import Path
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from datetime import datetime, timedelta, timezone
from src.reconcile import reconcile
class FakeObjects:
def __init__(self, items):
self.items = items
self.deleted = []
def objects(self):
return iter(self.items)
def delete(self, key):
self.deleted.append(key)
self.items = [(name, date) for name, date in self.items if name != key]
def put(self, key, content, content_type):
self.items.append((key, datetime.now(timezone.utc)))
class FakeStore:
def __init__(self):
now = datetime.now(timezone.utc)
self.metadata = {
"original/present": {"id": "present-id", "object_key": "original/present"},
"original/missing": {
"id": "missing-id",
"object_key": "original/missing",
"size_bytes": 4,
"sha256": hashlib.sha256(b"test").hexdigest(),
"content_type": "image/png",
},
}
self.objects = FakeObjects(
[
("original/present", now),
("original/stale-orphan", now - timedelta(hours=2)),
("original/new-orphan", now - timedelta(minutes=10)),
]
)
self.missing_incidents = set()
self.cleared_incidents = set()
def reconciliation_snapshot(self):
return self.metadata
def record_missing_object(self, attachment_id):
self.missing_incidents.add(attachment_id)
def clear_missing_object(self, attachment_id):
self.missing_incidents.discard(attachment_id)
self.cleared_incidents.add(attachment_id)
def main():
store = FakeStore()
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
assert dry_run["stale_orphan_candidates"] == 1
assert dry_run["missing_objects"] == 1
assert dry_run["deleted_orphans"] == 0
assert store.objects.deleted == [] and store.missing_incidents == set()
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
assert store.objects.deleted == ["original/stale-orphan"]
assert "original/new-orphan" not in store.objects.deleted
assert store.missing_incidents == {"missing-id"}
assert store.cleared_incidents == {"present-id"}
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
with tempfile.TemporaryDirectory() as backup:
restore_dir = Path(backup)
(restore_dir / "missing-id").write_bytes(b"test")
preview = reconcile(store, 3600, False, restore_dir)
assert preview["restorable_missing_objects"] == 1
assert store.missing_incidents == {"missing-id"}
restored = reconcile(store, 3600, True, restore_dir)
assert restored["restored_missing_objects"] == 1
assert store.missing_incidents == set()
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
assert store.objects.deleted == ["original/stale-orphan"]
assert store.missing_incidents == set()
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
if __name__ == "__main__":
main()
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
import tempfile
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
STORE_CODE = """
import sys
from pathlib import Path
from src.store import Store
store = Store()
store.open()
try:
action = sys.argv[1]
if action == "key":
attachment_id = sys.argv[2]
matches = [
key for key, row in store.reconciliation_snapshot().items()
if str(row["id"]) == attachment_id and row["reconcile_type"] == "original"
]
if len(matches) != 1:
raise RuntimeError("attachment metadata lookup failed")
print(matches[0])
elif action == "put":
key, file_path = sys.argv[2:4]
content = Path(file_path).read_bytes()
store.objects.put(key, content, "image/png", len(content))
elif action == "delete":
store.objects.delete(sys.argv[2])
elif action == "exists":
print("yes" if any(key == sys.argv[2] for key, _ in store.objects.objects()) else "no")
else:
raise RuntimeError("unsupported test operation")
finally:
store.close()
"""
PRIVATE_CHECK_CODE = """
import socket
import sys
import urllib.error
import urllib.request
try:
socket.getaddrinfo("s3-core", 8888)
except socket.gaierror:
print("PASS: SeaweedFS master/filer network is isolated")
else:
raise RuntimeError("API network can resolve the SeaweedFS core service")
for port in (8181, 9101):
try:
socket.create_connection(("s3", port), timeout=1).close()
except OSError:
continue
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
try:
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
except urllib.error.HTTPError as error:
if error.code == 403:
print("PASS: unauthenticated bucket access denied")
else:
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
else:
raise RuntimeError("private bucket allowed an unauthenticated GET")
"""
def docker_exec(container_id, *args):
return subprocess.run(
["docker", "exec", container_id, *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def reconciliation(container_id, *args):
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
return json.loads(result.stdout)
def object_exists(container_id, key):
return docker_exec(
container_id, "python3", "-c", STORE_CODE, "exists", key
).stdout.strip() == "yes"
def main():
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
attachment, content = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
containers = result.stdout.split()
check(len(containers) >= 2, "at least two running API replicas are required")
container_id = containers[0]
key = docker_exec(
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
).stdout.strip()
orphan_key = "original/" + str(uuid.uuid4())
with tempfile.TemporaryDirectory() as backup:
source = Path(backup) / "original.png"
source.write_bytes(content)
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
recovery_file = recovery_dir + "/" + attachment["id"]
orphan_created = False
orphan_removed = False
missing_deleted = False
missing_restored = False
try:
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
orphan_created = True
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
missing_deleted = True
preview = reconciliation(container_id)
check(
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
"dry-run missed injected inconsistencies",
)
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
immediate_apply = reconciliation(container_id, "--apply")
check(object_exists(container_id, orphan_key), "fresh orphan was deleted during the grace period")
docker_exec(container_id, "mkdir", "-p", recovery_dir)
subprocess.run(
["docker", "cp", str(source), container_id + ":" + recovery_file],
check=True,
)
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
missing_restored = True
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
check(object_exists(container_id, orphan_key), "recent orphan did not respect grace period")
print("Waiting for the 60-second orphan grace period.")
time.sleep(61)
cleanup_preview = reconciliation(container_id)
check(cleanup_preview["stale_orphan_candidates"] >= 1, "stale orphan was not identified after grace")
cleaned = reconciliation(container_id, "--apply")
orphan_removed = not object_exists(container_id, orphan_key)
check(orphan_removed, "injected stale orphan was not removed")
final_scan = reconciliation(container_id, "--apply")
check(
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
"repeat scan is not clean",
)
finally:
if missing_deleted and not missing_restored:
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
if orphan_created and not orphan_removed:
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
check("PASS:" in private, "unauthenticated S3 access check failed")
downloaded = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
print(
json.dumps(
{
"result": "PASS",
"dataset_objects_scanned": final_scan["objects_seen"],
"metadata_records_scanned": final_scan["metadata_seen"],
"scan_duration_ms": final_scan["duration_ms"],
"orphan_detected_and_removed": cleaned["deleted_orphans"],
"missing_detected_and_restored": restored["restored_missing_objects"],
"fresh_orphan_preserved_until_grace": True,
"message_attachment_download_checksum": "unchanged",
"private_bucket": "anonymous request denied",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Local transformation checks for the lab5 image worker."""
import hashlib
import io
import os
from pathlib import Path
import sys
os.environ.setdefault("S3_ACCESS_KEY_ID", "worker-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "worker-test-secret")
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from src.worker import build_variants
def main():
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
attachment_id = "00000000-0000-4000-8000-000000000001"
variants = build_variants(attachment_id, original, pipeline_version=1)
by_name = {item["name"]: item for item in variants}
assert set(by_name) == {"thumbnail", "crop", "optimized"}
assert (by_name["thumbnail"]["width"], by_name["thumbnail"]["height"]) == (256, 160)
assert (by_name["crop"]["width"], by_name["crop"]["height"]) == (128, 128)
assert by_name["optimized"]["content_type"] == "image/webp"
for item in variants:
content = item["content"]
assert len(content) == item["size_bytes"]
assert hashlib.sha256(content).hexdigest() == item["sha256"]
with Image.open(io.BytesIO(content)) as image:
assert image.size == (item["width"], item["height"])
newer = build_variants(attachment_id, original, pipeline_version=2)
assert all(f"/v2/" in item["object_key"] for item in newer)
assert all(f"/v1/" in item["object_key"] for item in variants)
assert by_name["thumbnail"]["sha256"] != {
item["name"]: item for item in newer
}["thumbnail"]["sha256"]
print("PASS: versioned thumbnail, crop, and optimized variants match dimensions and checksums")
if __name__ == "__main__":
main()
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Verify queued uploads and message links survive an API/worker restart."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def main():
args = arguments("Verify queued attachment recovery after API and worker restart")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
compose("stop", "worker")
worker_stopped = True
try:
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not accepted while worker was stopped")
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to message")
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
409,
actor=bob,
)
compose("restart", "api")
compose("start", "worker")
worker_stopped = False
client.call("GET", "/health/ready", schema="Health")
ready = client.wait_ready(bob, attachment)
original_bytes = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)["bytes"]
check(original_bytes == original, "recovered worker changed original bytes")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages",
actor=bob,
schema="MessagePage",
)["json"]
recovered_message = next(item for item in page["items"] if item["id"] == message["id"])
check(recovered_message["attachment_ids"] == [attachment["id"]], "message lost attachment link")
print(
json.dumps(
{
"result": "PASS",
"initial_status": attachment["status"],
"final_status": ready["status"],
"message_attachment_link_survived": True,
"original_sha256": hashlib.sha256(original_bytes).hexdigest(),
},
sort_keys=True,
)
)
finally:
if worker_stopped:
compose("start", "worker")
if __name__ == "__main__":
run(main)
+365
View File
@@ -0,0 +1,365 @@
#!/usr/bin/env python3
"""Repeat a paced synthetic message workload with telemetry off and sampled."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import platform
import re
import ssl
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
def command(args, capture=True):
result = subprocess.run(
args,
cwd=ROOT,
check=True,
text=True,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
)
return result.stdout.strip() if capture else ""
def set_env_value(text, name, value):
lines = text.splitlines()
prefix = name + "="
for index, line in enumerate(lines):
if line.startswith(prefix):
lines[index] = prefix + value
break
else:
lines.append(prefix + value)
return "\n".join(lines) + "\n"
def parse_env(text):
values = {}
for line in text.splitlines():
if line and not line.startswith("#") and "=" in line:
name, value = line.split("=", 1)
values[name] = value
return values
def wait_ready(url, ca_file=None, timeout=90):
deadline = time.monotonic() + timeout
context = ssl.create_default_context(cafile=ca_file)
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url + "/health/ready", context=context, timeout=2) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError, OSError):
pass
time.sleep(2)
raise RuntimeError("API did not become ready after the sampling mode change")
def parse_size(value):
match = re.fullmatch(r"\s*([\d.]+)\s*(B|kB|MB|GB|KiB|MiB|GiB)\s*", value)
if not match:
return None
factors = {
"B": 1,
"kB": 1000,
"MB": 1000**2,
"GB": 1000**3,
"KiB": 1024,
"MiB": 1024**2,
"GiB": 1024**3,
}
return int(float(match.group(1)) * factors[match.group(2)])
def docker_resources():
rows = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}};{{.CPUPerc}};{{.MemUsage}}",
]
)
cpu_total = 0.0
memory_total = 0
found = False
for row in rows.splitlines():
fields = row.split(";")
if len(fields) != 3 or not any(
name in fields[0] for name in ("-api-", "-worker-", "-publisher-", "-otel-collector-")
):
continue
cpu = re.match(r"\s*([\d.]+)%", fields[1])
memory = parse_size(fields[2].split("/", 1)[0])
if cpu and memory is not None:
cpu_total += float(cpu.group(1))
memory_total += memory
found = True
return {"cpu_percent_sum": round(cpu_total, 2), "memory_bytes_sum": memory_total} if found else None
def storage_sizes(compose):
paths = {
"prometheus": (["exec", "-T", "prometheus", "du", "-sb", "/prometheus"]),
"loki": (["exec", "-T", "loki", "du", "-sb", "/loki"]),
"tempo": (["exec", "-T", "tempo", "du", "-sb", "/var/tempo"]),
}
result = {}
for name, args in paths.items():
output = command(compose + args)
result[name] = int(output.split()[0])
return result
def tempo_message_traces(compose, start, end):
url = (
"http://127.0.0.1:3200/api/search?start=%d&end=%d&limit=2000"
"&tags=service.name%%3Dmessenger-api" % (start, end)
)
output = command(compose + ["exec", "-T", "tempo", "wget", "-qO-", url])
data = json.loads(output)
return [
item
for item in data.get("traces", [])
if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"
]
def mean_spans(compose, traces, sample_limit=20):
counts = []
for item in traces[:sample_limit]:
output = command(
compose
+ [
"exec",
"-T",
"tempo",
"wget",
"-qO-",
"http://127.0.0.1:3200/api/traces/" + item["traceID"],
]
)
data = json.loads(output)
count = sum(
len(scope.get("spans", []))
for batch in data.get("batches", [])
for scope in batch.get("instrumentationLibrarySpans", batch.get("scopeSpans", []))
)
counts.append(count)
return round(sum(counts) / len(counts), 2) if counts else 0.0
def run_load(base_url, ca_file, duration, concurrency, target_rps, alice, chat):
from client import Client, Failure
barrier = threading.Barrier(concurrency + 1)
ticket_lock = threading.Lock()
ticket = 0
latencies = []
failures = 0
results_lock = threading.Lock()
start_clock = time.monotonic()
start_epoch = int(time.time())
deadline = start_clock + duration
def worker(worker_id):
nonlocal ticket, failures
client = Client(base_url, ca_file, timeout=5)
local_latencies = []
local_failures = 0
barrier.wait()
while True:
with ticket_lock:
current = ticket
ticket += 1
due = start_clock + current / target_rps
if due >= deadline:
break
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
started = time.monotonic()
try:
client.message(alice, chat, "synthetic-track1-%d-%d" % (worker_id, current))
local_latencies.append(time.monotonic() - started)
except Failure:
local_failures += 1
with results_lock:
latencies.extend(local_latencies)
failures += local_failures
samples = []
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = [pool.submit(worker, worker_id) for worker_id in range(concurrency)]
barrier.wait()
while any(not future.done() for future in futures):
sample = docker_resources()
if sample is not None:
samples.append(sample)
time.sleep(2)
for future in futures:
future.result()
elapsed = time.monotonic() - start_clock
sorted_latencies = sorted(latencies)
def percentile(value):
if not sorted_latencies:
return None
index = max(0, math.ceil(value * len(sorted_latencies)) - 1)
return round(sorted_latencies[index] * 1000, 3)
total = len(sorted_latencies) + failures
report = {
"workload": "paced synthetic POST messages; no automatic retries",
"requested_duration_seconds": duration,
"target_rps": target_rps,
"elapsed_seconds": round(elapsed, 3),
"concurrency": concurrency,
"successful_requests": len(sorted_latencies),
"failed_requests": failures,
"error_ratio": round(failures / total, 6) if total else None,
"successful_rps": round(len(sorted_latencies) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(0.50),
"p95": percentile(0.95),
"p99": percentile(0.99),
},
"docker_resource_samples": len(samples),
}
if samples:
report["docker_resources"] = {
"mean_cpu_percent_sum": round(
sum(item["cpu_percent_sum"] for item in samples) / len(samples), 2
),
"peak_cpu_percent_sum": max(item["cpu_percent_sum"] for item in samples),
"peak_memory_bytes_sum": max(item["memory_bytes_sum"] for item in samples),
"services": ["api", "worker", "publisher", "otel-collector"],
}
return report, start_epoch, int(time.time())
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--base-url", default=None)
parser.add_argument("--ca-file", default=None)
parser.add_argument("--duration", type=int, default=15)
parser.add_argument("--concurrency", type=int, default=2)
parser.add_argument("--target-rps", type=int, default=20)
parser.add_argument("--warmup-seconds", type=int, default=5)
parser.add_argument("--output", default="evidence/track1-observability-cost.json")
args = parser.parse_args()
if not 1 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.target_rps < 1:
parser.error("duration must be 1..3600, concurrency 1..128, and target-rps positive")
env_path = ROOT / args.env_file
if not env_path.is_file():
parser.error("create the local .env from .env.example and generate secrets first")
original = env_path.read_text(encoding="utf-8")
values = parse_env(original)
base_url = args.base_url or "https://localhost:" + values.get("APP_PORT", "8443")
compose = ["docker", "compose", "--env-file", str(env_path)]
output_path = ROOT / args.output
output_path.parent.mkdir(parents=True, exist_ok=True)
results = []
configurations = [
("off", "true", "1.0"),
("sample-100-percent", "false", "1.0"),
("sample-25-percent", "false", "0.25"),
("sample-10-percent", "false", "0.1"),
]
try:
for name, disabled, ratio in configurations:
changed = set_env_value(original, "OTEL_SDK_DISABLED", disabled)
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
changed = set_env_value(changed, "OTEL_TRACES_SAMPLER_ARG", ratio)
env_path.write_text(changed, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
wait_ready(base_url, args.ca_file)
time.sleep(args.warmup_seconds)
from client import Client
alice, _, _, chat = Client(base_url, args.ca_file, timeout=10).bootstrap()
time.sleep(2)
before = storage_sizes(compose)
load, start, end = run_load(
base_url, args.ca_file, args.duration, args.concurrency, args.target_rps, alice, chat
)
time.sleep(15)
after = storage_sizes(compose)
traces = tempo_message_traces(compose, start - 5, end + 20)
spans_per_trace = mean_spans(compose, traces)
results.append(
{
"configuration": name,
"sdk_disabled": disabled == "true",
"trace_sampling_ratio": float(ratio),
"load": load,
"window_unix_seconds": {"start": start, "end": end},
"sampled_message_trace_count": len(traces),
"estimated_spans_per_message_trace": spans_per_trace,
"persistent_storage_bytes_delta": {
signal: after[signal] - before[signal] for signal in before
},
}
)
baseline = next(
item["estimated_spans_per_message_trace"]
for item in results
if item["configuration"] == "sample-100-percent"
)
for item in results:
expected_spans = item["load"]["successful_requests"] * baseline
observed_spans = item["sampled_message_trace_count"] * item["estimated_spans_per_message_trace"]
item["estimated_spans_not_recorded"] = round(max(0.0, expected_spans - observed_spans), 2)
report = {
"experiment": "telemetry overhead under fixed-rate synthetic POST load",
"hypothesis": "Trace sampling lowers API/worker CPU and Tempo ingest; metrics and logs remain enabled.",
"method": {
"duration_seconds": args.duration,
"concurrency": args.concurrency,
"target_rps": args.target_rps,
"warmup_seconds": args.warmup_seconds,
"telemetry_flush_wait_seconds": 15,
"hardware": {
"platform": platform.platform(),
"processor": platform.processor() or "unspecified",
"logical_cpus": os.cpu_count(),
},
"sampled_span_estimate": "The first 20 matching message traces per mode are opened in Tempo; the resulting mean spans per trace is extrapolated to all sampled messages.",
"storage_method": "Filesystem byte deltas in Prometheus, Loki and Tempo named volumes; this includes indexes/WAL and is not network wire size.",
},
"limitations": [
"The generator paces arrivals but has finite concurrency and can catch up after delays; it is not a production open-loop load model.",
"Database history grows between modes; all modes use identical duration and concurrency.",
"Sampling is head-based and random: errors and rare slow requests can be omitted along with successful requests.",
"Span loss is an estimate based on sampled message traces, not Collector exporter delivery counters.",
"CPU and memory are sampled Docker container totals for API, worker, publisher and Collector; the monitor adds equal overhead to all modes.",
],
"results": results,
"balanced_configuration": "sample-25-percent",
}
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
finally:
env_path.write_text(original, encoding="utf-8")
command(compose + ["up", "-d", "--force-recreate", "api", "publisher", "worker"])
if __name__ == "__main__":
main()
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
import threading
import time
import uuid
from io import BytesIO
from pathlib import Path
import sys
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
def usage_bytes(line):
match = USAGE_RE.search(line)
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
def api_containers():
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
ids = result.stdout.split()
check(len(ids) >= 2, "at least two running API containers are required")
return ids
def sample_once(ids):
output = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
check=True,
capture_output=True,
text=True,
).stdout.splitlines()
values = [usage_bytes(line) for line in output]
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
return max(values)
def main():
args = arguments("Bounded-memory ten-client upload experiment")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
width = height = 1400
content = BytesIO()
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
content, format="PNG", compress_level=0
)
payload = content.getvalue()
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
ids = api_containers()
baseline = sample_once(ids)
samples = []
stats_process = subprocess.Popen(
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
def sampler():
if stats_process.stdout is None:
return
for line in stats_process.stdout:
value = usage_bytes(line)
if value:
samples.append(value)
sampling_thread = threading.Thread(target=sampler, daemon=True)
sampling_thread.start()
count = 10
barrier = threading.Barrier(count)
def upload(index):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=15)
return worker.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
202,
actor=alice,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(upload, range(count)))
finally:
stopped_by_test = False
if stats_process.poll() is None:
stats_process.kill()
stopped_by_test = True
try:
stats_process.wait(timeout=5)
except subprocess.TimeoutExpired:
stats_process.kill()
stats_process.wait(timeout=5)
sampling_thread.join(timeout=5)
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
check(
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
"docker stats sampling process failed (exit "
+ str(stats_process.returncode)
+ "): "
+ stats_error.strip()[:300],
)
check(bool(samples), "docker stats produced no memory samples")
peak = max([baseline] + samples)
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
print(
json.dumps(
{
"uploads": count,
"concurrency": count,
"bytes_per_upload": len(payload),
"duration_seconds": round(time.monotonic() - started, 3),
"api_containers": len(ids),
"sample_count": len(samples) // len(ids),
"baseline_max_container_bytes": baseline,
"peak_max_container_bytes": peak,
"peak_delta_bytes": max(0, peak - baseline),
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+25 -12
View File
@@ -1,12 +1,21 @@
# Публичные проверки
# Проверки lab7
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
HTTP проверки выполняются против запущенного Compose стенда; они не создают его автоматически. Клиенты Python используют стандартную библиотеку; сервер и unit tests требуют runtime зависимостей из `requirements.txt`. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
```sh
make check
make test BASE_URL=http://localhost:8080
# без make:
python3 tests/smoke.py --base-url http://localhost:8080
make test BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-security BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-primitives
make test-internal-mtls
make test-login-limit BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-openbao-acl PYTHON=.venv/bin/python
make test-worker-isolation
make test-worker-rotation PYTHON=.venv/bin/python ROOT_TOKEN_FILE=/secure/path/openbao-root-token
make test-leaf-rotation PYTHON=.venv/bin/python
make test-telemetry-redaction PYTHON=.venv/bin/python BASE_URL=https://localhost:8443 CA_FILE="$PWD/secrets/pki/ca.crt"
make test-argon-budget
.venv/bin/pip-audit -r requirements.txt
```
`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite.
@@ -15,10 +24,9 @@ python3 tests/smoke.py --base-url http://localhost:8080
## Проверка сохранности, lab2+
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
`scripts/restart.sh` перезапускает API и используемые stores, сохраняя volumes и тестовые данные.
```sh
chmod +x scripts/restart.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
```
@@ -26,10 +34,9 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh
## Проверка отказа, lab3+
Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо.
`scripts/stop-one.sh` сопоставляет `TARGET_INSTANCE_ID` с hostname API container и останавливает только его. Скрипт оставляет реплику остановленной; восстановление выполняйте отдельно.
```sh
chmod +x scripts/stop-one.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
```
@@ -50,8 +57,14 @@ make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
```
Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях.
Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. `make test-internal-mtls` проверяет внутренний API listener из контейнера API: без клиентского сертификата, с недоверенной CA и с недопустимой URI SAN соединение отвергается, а LB identity проходит. `make test-login-limit` проверяет ответ 429 и Retry-After после серии неуспешных входов. Клиентский сертификат внешнему REST-клиенту не требуется: mTLS находится на связи LB→API.
## Что ещё остаётся доказать
`make test-openbao-acl` проверяет все сервисные AppRole: каждая identity читает собственный KV path и получает 403 на paths остальных identities; также проверяется ограниченный TTL токена. `make test-argon-budget` измеряет Argon2id verify latency, burst capacity и число одновременно занятых hash slots внутри API container с его CPU/RAM limits. Эти результаты характеризуют password hashing guard, а не end-to-end capacity стенда. `pip-audit -r requirements.txt` сканирует закреплённые Python runtime-зависимости по advisory database.
Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
`make test-worker-isolation` запускается внутри worker container: проверяет ограниченную PostgreSQL-роль, запрет чтения/перечисления защищённых S3 buckets, разрешённые staging/originals/variants операции и запрет публикации worker в RabbitMQ. Временные probe objects удаляются после теста.
`make test-worker-rotation ROOT_TOKEN_FILE=...` требует запущенный и распечатанный OpenBao и локальный путь к operator root-token file вне Git. Скрипт создаёт attachment при остановленном worker, выпускает новый SecretID, отзывает прежний, доказывает отказ старого credential, пересоздаёт только worker и проверяет `queued → ready`, checksum original и message link. Root token и SecretID не печатаются. Запуск создаёт синтетические записи и заменяет локальный worker SecretID.
`make test-leaf-rotation` выпускает и устанавливает новый LB client leaf с прежней URI SAN, пересоздаёт API и HAProxy без build, проверяет отказ старого сертификата, приём нового и неизменность Docker image IDs. `make test-telemetry-redaction` генерирует синтетические credentials и сообщение, затем сканирует API log entries в Loki и трассы Tempo на отсутствие этих значений.
Smoke и security-проверки рассчитаны на localhost Compose стенд, не на production. Нагрузочный benchmark, persistence hook и failover API выполняются отдельными командами выше; их машинные результаты записываются в `evidence/`.
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
import argparse
import json
import math
import resource
import statistics
from pathlib import Path
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src import security
def percentile(values, fraction):
ordered = sorted(values)
return ordered[max(0, math.ceil(len(ordered) * fraction) - 1)]
def main():
parser = argparse.ArgumentParser(description="Measure Argon2id latency and bounded concurrency.")
parser.add_argument("--samples", type=int, default=20)
parser.add_argument("--concurrency", type=int, default=12)
args = parser.parse_args()
if args.samples < 1 or args.concurrency < 1:
parser.error("samples and concurrency must be positive")
encoded = security.hash_password("argon2-budget-synthetic-password")
durations = []
for _ in range(args.samples):
started = time.perf_counter()
if not security.verify_password("argon2-budget-synthetic-password", encoded):
raise RuntimeError("Argon2id self-verification failed")
durations.append(time.perf_counter() - started)
active = 0
peak_active = 0
lock = threading.Lock()
original_slots = security._PASSWORD_HASH_SLOTS
class TrackingSlots:
def acquire(self, timeout=None):
nonlocal active, peak_active
acquired = original_slots.acquire(timeout=timeout)
if acquired:
with lock:
active += 1
peak_active = max(peak_active, active)
return acquired
def release(self):
nonlocal active
with lock:
active -= 1
original_slots.release()
security._PASSWORD_HASH_SLOTS = TrackingSlots()
def verify_once(_):
try:
return security.verify_password("argon2-budget-synthetic-password", encoded)
except security.PasswordHashCapacityError:
return False
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
burst_results = list(executor.map(verify_once, range(args.concurrency)))
print(json.dumps({
"argon2_memory_kib": security._ARGON2_MEMORY_KIB,
"argon2_iterations": security._ARGON2_ITERATIONS,
"argon2_parallelism": security._ARGON2_PARALLELISM,
"hash_slots_per_process": security._PASSWORD_HASH_CONCURRENCY,
"hash_wait_seconds": security._PASSWORD_HASH_WAIT_SECONDS,
"sequential_samples": args.samples,
"verify_latency_ms": {
"p50": round(statistics.median(durations) * 1000, 3),
"p95": round(percentile(durations, 0.95) * 1000, 3),
"max": round(max(durations) * 1000, 3),
},
"burst_requests": args.concurrency,
"burst_successes": sum(burst_results),
"burst_capacity_rejections": args.concurrency - sum(burst_results),
"peak_active_hashes": peak_active,
"max_rss_platform_units": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
"platform": sys.platform,
}, sort_keys=True))
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Verify API listener client-certificate requirements from inside an API container."""
import ssl
import http.client
import urllib.error
import urllib.request
HOST = "https://127.0.0.1:8080/health/ready"
CA = "/run/secrets/ca_crt"
def context(client_cert=None, client_key=None):
result = ssl.create_default_context(cafile=CA)
if client_cert:
result.load_cert_chain(client_cert, client_key)
return result
def rejected(client_cert=None, client_key=None):
try:
with urllib.request.urlopen(HOST, context=context(client_cert, client_key), timeout=5) as response:
response.read(1024)
except (urllib.error.URLError, ssl.SSLError, http.client.HTTPException, OSError):
return
raise AssertionError("API accepted a request with an unauthorized client identity")
def main():
rejected()
rejected("/run/secrets/wrong_client_cert", "/run/secrets/wrong_client_key")
rejected("/run/secrets/foreign_client_cert", "/run/secrets/foreign_client_key")
with urllib.request.urlopen(
HOST,
context=context("/run/secrets/lb_tls_cert", "/run/secrets/lb_tls_key"),
timeout=5,
) as response:
if response.status != 200:
raise AssertionError(f"authorized LB identity returned HTTP {response.status}")
print("PASS: missing, foreign-CA and non-LB identities rejected; LB identity accepted.")
if __name__ == "__main__":
main()
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""Rotate the LB client leaf, pin the replacement, and prove the prior leaf is rejected."""
import hashlib
import json
import os
from pathlib import Path
import ssl
import subprocess
import tempfile
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
PKI_DIR = ROOT / "secrets" / "pki"
CA_CERT = PKI_DIR / "ca.crt"
CA_KEY = PKI_DIR / "ca.key"
CERT_PATH = PKI_DIR / "lb-client.crt"
KEY_PATH = PKI_DIR / "lb-client.key"
BUNDLE_PATH = PKI_DIR / "lb-client.pem"
API_PROBE = """
import json, ssl, sys, tempfile, urllib.request, urllib.error
value = json.load(sys.stdin)
with tempfile.TemporaryDirectory() as directory:
cert = directory + "/client.crt"
key = directory + "/client.key"
open(cert, "w").write(value["cert"])
open(key, "w").write(value["key"])
context = ssl.create_default_context(cafile="/run/secrets/ca_crt")
context.load_cert_chain(cert, key)
try:
with urllib.request.urlopen("https://127.0.0.1:8080/health/ready", context=context, timeout=5) as response:
result = {"accepted": response.status == 200}
except (urllib.error.URLError, ssl.SSLError, OSError):
result = {"accepted": False}
print(json.dumps(result))
"""
def run(command, *, input_text=None):
return subprocess.run(
command,
cwd=ROOT,
input=input_text,
text=True,
check=True,
capture_output=True,
)
def compose(*args, input_text=None):
return run(["docker", "compose", *args], input_text=input_text)
def write_atomic(path: Path, content: bytes, mode: int) -> None:
temporary = path.with_suffix(path.suffix + ".rotation-tmp")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
with os.fdopen(descriptor, "wb") as output:
output.write(content)
os.chmod(temporary, mode)
os.replace(temporary, path)
def probe(cert: bytes, key: bytes) -> bool:
payload = json.dumps(
{
"cert": cert.decode("ascii"),
"key": key.decode("ascii"),
}
)
result = compose("exec", "-T", "api", "python3", "-c", API_PROBE, input_text=payload)
return json.loads(result.stdout)["accepted"]
def image_ids() -> set:
container_ids = compose("ps", "-q", "api").stdout.split()
if len(container_ids) != 2:
raise RuntimeError("expected exactly two running API replicas")
return {
run(["docker", "inspect", "--format", "{{.Image}}", container_id]).stdout.strip()
for container_id in container_ids
}
def wait_healthy(timeout: int = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
containers = compose("ps", "-q", "api").stdout.split()
healthy = len(containers) == 2
for container_id in containers:
state = json.loads(
run(["docker", "inspect", "--format", "{{json .State.Health}}", container_id]).stdout
)
healthy = healthy and state.get("Status") == "healthy"
lb = compose("ps", "-q", "load_balancer").stdout.split()
healthy = healthy and len(lb) == 1
if healthy:
state = json.loads(
run(["docker", "inspect", "--format", "{{json .State.Health}}", lb[0]]).stdout
)
healthy = state.get("Status") == "healthy"
if healthy:
return
time.sleep(1)
raise RuntimeError("API and load balancer did not become healthy after certificate rotation")
def create_replacement(directory: Path) -> tuple[bytes, bytes, bytes]:
request_path = directory / "lb-client.csr"
cert_path = directory / "lb-client.crt"
key_path = directory / "lb-client.key"
extensions_path = directory / "lb-client.ext"
subprocess.run(
[
"openssl", "req", "-new", "-newkey", "rsa:2048", "-nodes",
"-keyout", str(key_path), "-out", str(request_path),
"-subj", "/CN=lab7-load-balancer",
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
extensions_path.write_text(
"basicConstraints=critical,CA:FALSE\n"
"keyUsage=critical,digitalSignature,keyEncipherment\n"
"extendedKeyUsage=clientAuth\n"
"subjectAltName=URI:spiffe://mtusi2026lab7/load-balancer\n",
encoding="ascii",
)
subprocess.run(
[
"openssl", "x509", "-req", "-in", str(request_path),
"-CA", str(CA_CERT), "-CAkey", str(CA_KEY), "-CAcreateserial",
"-out", str(cert_path), "-days", "30", "-sha256",
"-extfile", str(extensions_path),
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
cert = cert_path.read_bytes()
key = key_path.read_bytes()
return cert, key, cert + key
def main() -> None:
if not CA_CERT.is_file() or not CA_KEY.is_file():
raise RuntimeError("local CA certificate and operator-held private key are required")
old_cert = CERT_PATH.read_bytes()
old_key = KEY_PATH.read_bytes()
old_bundle = BUNDLE_PATH.read_bytes()
before_images = image_ids()
if not probe(old_cert, old_key):
raise RuntimeError("current LB certificate was not accepted before rotation")
replaced = False
try:
with tempfile.TemporaryDirectory(prefix="lab7-lb-leaf-") as directory_name:
new_cert, new_key, new_bundle = create_replacement(Path(directory_name))
new_fingerprint = hashlib.sha256(ssl.PEM_cert_to_DER_cert(new_cert.decode("ascii"))).hexdigest()
write_atomic(CERT_PATH, new_cert, 0o644)
write_atomic(KEY_PATH, new_key, 0o600)
write_atomic(BUNDLE_PATH, new_bundle, 0o600)
replaced = True
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
wait_healthy()
if probe(old_cert, old_key):
raise RuntimeError("previous LB certificate remains authorized after rotation")
if not probe(new_cert, new_key):
raise RuntimeError("replacement LB certificate was not accepted")
context = ssl.create_default_context(cafile=str(CA_CERT))
with urllib.request.urlopen(
"https://localhost:8443/health/ready",
context=context,
timeout=10,
) as response:
if response.status != 200:
raise RuntimeError("HTTPS ingress did not recover after leaf rotation")
after_images = image_ids()
if before_images != after_images:
raise RuntimeError("container image changed; expected rotation without rebuild")
print(
json.dumps(
{
"result": "PASS",
"old_leaf_rejected": True,
"new_leaf_accepted": True,
"https_ingress_healthy": True,
"images_unchanged": True,
"new_leaf_sha256": new_fingerprint,
},
sort_keys=True,
)
)
except Exception:
if replaced:
write_atomic(CERT_PATH, old_cert, 0o644)
write_atomic(KEY_PATH, old_key, 0o600)
write_atomic(BUNDLE_PATH, old_bundle, 0o600)
compose("up", "-d", "--no-deps", "--force-recreate", "api", "load_balancer")
raise
if __name__ == "__main__":
main()
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
import base64
import uuid
from client import Client, arguments, check, run
def main():
args = arguments("Verify the shared login rate limit")
client = Client(args.base_url, args.ca_file)
username = "limit_" + uuid.uuid4().hex[:16]
client.call(
"POST",
"/api/v1/users",
201,
body={"username": username, "display_name": "Rate limit test", "password": "Correct-Password-123"},
schema="User",
)
invalid = base64.b64encode((username + ":Wrong-Password-123").encode()).decode("ascii")
for _ in range(5):
client.call(
"POST",
"/api/v1/auth/sessions",
401,
headers={"Authorization": "Basic " + invalid},
)
result = client.call(
"POST",
"/api/v1/auth/sessions",
429,
headers={"Authorization": "Basic " + invalid},
)
check(result["json"]["error"]["code"] == "rate_limited", "expected rate_limited response")
check(int(result["headers"].get("Retry-After", "0")) > 0, "missing positive Retry-After")
print("PASS: sixth failed login in the window is rejected with 429 and Retry-After.")
if __name__ == "__main__":
run(main)
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
import argparse
import json
import ssl
import urllib.error
import urllib.request
from pathlib import Path
SERVICES = ("api", "worker", "publisher", "migrate", "reconciler")
class BaoClient:
def __init__(self, address: str, ca_file: str):
self.address = address.rstrip("/")
self.context = ssl.create_default_context(cafile=ca_file)
def request(self, method: str, path: str, payload=None, token: str = ""):
body = json.dumps(payload).encode("utf-8") if payload is not None else None
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
request = urllib.request.Request(
self.address + "/v1/" + path,
data=body,
headers=headers,
method=method,
)
try:
with urllib.request.urlopen(request, context=self.context, timeout=5) as response:
return response.status, json.loads(response.read(1024 * 1024))
except urllib.error.HTTPError as error:
return error.code, {}
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error:
raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None
def main() -> None:
parser = argparse.ArgumentParser(description="Check service AppRole path isolation.")
parser.add_argument("--address", default="https://localhost:8200")
parser.add_argument("--ca-file", required=True)
parser.add_argument("--credential-dir", default="secrets/openbao")
args = parser.parse_args()
credential_dir = Path(args.credential_dir)
client = BaoClient(args.address, args.ca_file)
for service in SERVICES:
role_id = (credential_dir / f"{service}_role_id").read_text(encoding="utf-8").strip()
secret_id = (credential_dir / f"{service}_secret_id").read_text(encoding="utf-8").strip()
status, response = client.request(
"POST",
"auth/approle/login",
{"role_id": role_id, "secret_id": secret_id},
)
token = response.get("auth", {}).get("client_token")
if status != 200 or not token:
raise RuntimeError(f"{service} AppRole authentication failed with HTTP {status}")
lease_duration = response.get("auth", {}).get("lease_duration")
if not isinstance(lease_duration, int) or not 1 <= lease_duration <= 3600:
raise RuntimeError(f"{service} AppRole returned an invalid token TTL")
own_status, _ = client.request(
"GET",
f"secret/data/messenger/{service}",
token=token,
)
if own_status != 200:
raise RuntimeError(f"{service} cannot read its own scoped KV path (HTTP {own_status})")
for other in SERVICES:
if other == service:
continue
denied_status, _ = client.request(
"GET",
f"secret/data/messenger/{other}",
token=token,
)
if denied_status != 403:
raise RuntimeError(
f"{service} AppRole accessed {other} KV path (HTTP {denied_status})"
)
lookup_status, lookup_response = client.request(
"GET",
"auth/token/lookup-self",
token=token,
)
token_data = lookup_response.get("data", {})
if lookup_status != 200 or not 1 <= token_data.get("ttl", 0) <= 3600:
raise RuntimeError(
f"{service} AppRole token TTL is not bounded (HTTP {lookup_status})"
)
print("PASS: each AppRole reads its own KV path and is denied all other service paths.")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -36,7 +36,7 @@ def main():
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
check('.example.' not in script.name,'copy and implement the example hook first')
c = Client(args.base_url,args.ca_file,timeout=2)
c = Client(args.base_url,args.ca_file,timeout=15)
alice,bob,eve,chat = c.bootstrap()
sent = c.message(alice,chat,'Сохранить при отказе')
original = sent['json']
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Verify synthetic credentials and message bodies are absent from Loki logs and Tempo spans."""
import base64
import json
from pathlib import Path
import re
import subprocess
import time
import uuid
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
TELEMETRY_PROBE = r"""
import json, ssl, sys, time, urllib.parse, urllib.request
probe = json.load(sys.stdin)
context = ssl.create_default_context(cafile="/run/secrets/ca_crt")
def request(url):
with urllib.request.urlopen(url, context=context, timeout=8) as response:
return json.load(response)
log_query = '{service_name="messenger-api"}'
deadline = time.monotonic() + 30
trace_ids = []
log_entries = []
while time.monotonic() < deadline:
params = urllib.parse.urlencode({"query": log_query, "start": probe["start_ns"], "end": str(time.time_ns()), "limit": "1000"})
logs = request("https://loki:3100/loki/api/v1/query_range?" + params)
log_entries = [
{"labels": stream.get("stream", {}), "line": line}
for stream in logs.get("data", {}).get("result", [])
for _, line in stream.get("values", [])
]
search = urllib.parse.urlencode({"limit": "1000", "start": probe["start"], "end": str(int(time.time())), "tags": "service.name=messenger-api"})
found = request("https://tempo:3200/api/search?" + search)
trace_ids = [item["traceID"] for item in found.get("traces", []) if item.get("rootTraceName") == "POST /api/v1/chats/{chat_id}/messages"]
if trace_ids and log_entries:
break
time.sleep(1)
if not trace_ids:
raise RuntimeError("no API message trace appeared in Tempo within the deadline")
payloads = [json.dumps(log_entries, ensure_ascii=False)]
for trace_id in trace_ids:
payloads.append(json.dumps(request("https://tempo:3200/api/traces/" + trace_id), ensure_ascii=False))
leaked = [value for value in probe["values"] if any(value in payload for payload in payloads)]
print(json.dumps({"trace_count": len(trace_ids), "log_entries": len(log_entries), "leak_count": len(leaked)}))
"""
def main() -> None:
args = arguments("Verify request secrets and message bodies are excluded from telemetry")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
canary = "telemetry-canary-" + uuid.uuid4().hex
username = "u_probe_" + uuid.uuid4().hex[:12]
password = "Password-" + uuid.uuid4().hex
start_time = time.time()
user = client.call(
"POST",
"/api/v1/users",
201,
body={"username": username, "display_name": canary, "password": password},
schema="User",
)["json"]
auth_value = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
session = client.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + auth_value},
schema="Session",
)["json"]
check(session["user"]["id"] == user["id"], "telemetry probe user session did not authenticate")
client.call(
"GET",
"/api/v1/users/me",
401,
headers={"Authorization": "Bearer " + canary},
)
client.message(alice, chat, canary)
values = [canary, username, password, auth_value, session["token"], alice["token"]]
probe = {
"values": values,
"start": str(max(0, int(start_time) - 5)),
"start_ns": str(max(0, int(start_time - 5) * 1_000_000_000)),
}
result = subprocess.run(
[
"docker", "compose", "exec", "-T", "api",
"python3", "-c", TELEMETRY_PROBE,
],
cwd=ROOT,
input=json.dumps(probe),
text=True,
capture_output=True,
check=True,
)
telemetry = json.loads(result.stdout)
check(telemetry["trace_count"] > 0, "no API traces were scanned")
check(telemetry["leak_count"] == 0, "synthetic credentials or message content appeared in telemetry")
print(
json.dumps(
{
"result": "PASS",
"synthetic_values_checked": len(values),
"api_traces_scanned": telemetry["trace_count"],
"log_entries_scanned": telemetry["log_entries"],
"leaks": 0,
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)
+54
View File
@@ -0,0 +1,54 @@
import base64
import hashlib
import unittest
from unittest.mock import patch
from src.security import (
PasswordHashCapacityError,
allowed_client_identity,
hash_password,
password_hash_needs_upgrade,
verify_password,
)
class PasswordHashTests(unittest.TestCase):
def test_argon2id_verifies_unicode_password(self):
encoded = hash_password("корректный-password-123")
self.assertTrue(encoded.startswith("$argon2id$"))
self.assertIn("m=19456,t=2,p=1", encoded)
self.assertTrue(verify_password("корректный-password-123", encoded))
self.assertFalse(verify_password("другой-password-123", encoded))
self.assertFalse(password_hash_needs_upgrade(encoded))
oversized = "$argon2id$v=19$m=999999,t=2,p=1$c2FsdA$ZGlnaWVzdA"
self.assertFalse(verify_password("attacker-password-123", oversized))
def test_legacy_scrypt_hash_is_verified_and_marked_for_upgrade(self):
password = "legacy-password-123"
salt = b"lab6-test-salt!!"
digest = hashlib.scrypt(
password.encode(), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
)
stored = "scrypt$32768$8$1${}${}".format(
base64.urlsafe_b64encode(salt).decode(),
base64.urlsafe_b64encode(digest).decode(),
)
self.assertTrue(verify_password(password, stored))
self.assertTrue(password_hash_needs_upgrade(stored))
def test_only_lb_service_identity_is_authorized(self):
trusted = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/load-balancer")]}
other = {"subjectAltName": [("URI", "spiffe://mtusi2026lab7/other-service")]}
self.assertTrue(allowed_client_identity(trusted))
self.assertFalse(allowed_client_identity(other))
self.assertFalse(allowed_client_identity({}))
def test_password_hash_capacity_fails_fast(self):
with patch("src.security._PASSWORD_HASH_SLOTS") as slots:
slots.acquire.return_value = False
with self.assertRaises(PasswordHashCapacityError):
verify_password("password", "$argon2id$malformed")
if __name__ == "__main__":
unittest.main()
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""Revoke the worker AppRole SecretID and recover a queued attachment with a replacement."""
import json
import os
from pathlib import Path
import ssl
import subprocess
import urllib.error
import urllib.request
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
SECRET_DIR = ROOT / "secrets" / "openbao"
class BaoClient:
def __init__(self, ca_file: str, root_token_file: str):
self.address = "https://localhost:8200"
self.context = ssl.create_default_context(cafile=ca_file)
self.root_token = Path(root_token_file).read_text(encoding="utf-8").strip()
check(bool(self.root_token), "operator root-token file is empty")
def request(self, method: str, path: str, payload=None, token=None):
body = json.dumps(payload).encode("utf-8") if payload is not None else None
headers = {"Content-Type": "application/json"}
if token is not None:
if token:
headers["X-Vault-Token"] = token
elif self.root_token:
headers["X-Vault-Token"] = self.root_token
request = urllib.request.Request(
self.address + "/v1/" + path,
data=body,
headers=headers,
method=method,
)
try:
with urllib.request.urlopen(request, context=self.context, timeout=8) as response:
data = response.read(1024 * 1024)
return response.status, json.loads(data) if data else {}
except urllib.error.HTTPError as error:
return error.code, {}
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error:
raise RuntimeError(f"OpenBao request failed ({type(error).__name__})") from None
def compose(*args) -> None:
subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
def save_secret_id(value: str) -> None:
path = SECRET_DIR / "worker_secret_id"
temporary = path.with_suffix(".tmp")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
output.write(value + "\n")
os.replace(temporary, path)
def rotate_worker_secret(bao: BaoClient) -> None:
role_id = (SECRET_DIR / "worker_role_id").read_text(encoding="utf-8").strip()
old_secret_id = (SECRET_DIR / "worker_secret_id").read_text(encoding="utf-8").strip()
check(bool(role_id) and bool(old_secret_id), "worker AppRole credential files are empty")
status, created = bao.request(
"POST",
"auth/approle/role/worker/secret-id",
{},
)
new_secret_id = created.get("data", {}).get("secret_id")
check(status == 200 and isinstance(new_secret_id, str), "OpenBao did not issue a replacement SecretID")
save_secret_id(new_secret_id)
status, login = bao.request(
"POST",
"auth/approle/login",
{"role_id": role_id, "secret_id": new_secret_id},
token="",
)
token = login.get("auth", {}).get("client_token")
check(status == 200 and isinstance(token, str), "replacement worker SecretID did not authenticate")
status, _ = bao.request(
"GET",
"secret/data/messenger/worker",
token=token,
)
check(status == 200, "replacement worker identity cannot read its own secret path")
status, _ = bao.request(
"POST",
"auth/approle/role/worker/secret-id/destroy",
{"secret_id": old_secret_id},
)
check(status == 204, "OpenBao did not revoke the previous worker SecretID")
status, _ = bao.request(
"POST",
"auth/approle/login",
{"role_id": role_id, "secret_id": old_secret_id},
token="",
)
check(status == 400, "revoked worker SecretID still authenticates")
def main() -> None:
def configure(parser):
parser.add_argument("--root-token-file", required=True)
args = arguments("Verify queued attachment recovery after worker SecretID rotation", configure)
ca_file = args.ca_file or str(ROOT / "secrets" / "pki" / "ca.crt")
client = Client(args.base_url, ca_file)
bao = BaoClient(ca_file, args.root_token_file)
alice, bob, _, chat = client.bootstrap()
worker_stopped = False
try:
compose("stop", "worker")
worker_stopped = True
attachment, original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not queued while the worker was stopped")
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(message["attachment_ids"] == [attachment["id"]], "queued attachment was not linked to its message")
rotate_worker_secret(bao)
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
worker_stopped = False
ready = client.wait_ready(bob, attachment)
original_bytes = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
actor=bob,
)["bytes"]
check(original_bytes == original, "rotated worker changed the original attachment bytes")
page = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages",
actor=bob,
schema="MessagePage",
)["json"]
recovered = next(item for item in page["items"] if item["id"] == message["id"])
check(recovered["attachment_ids"] == [attachment["id"]], "message lost its attachment link")
print(
json.dumps(
{
"result": "PASS",
"old_worker_secret_id_revoked": True,
"replacement_worker_secret_id_authenticated": True,
"attachment_state": [attachment["status"], ready["status"]],
"original_sha256_preserved": True,
"message_attachment_link_preserved": True,
},
sort_keys=True,
)
)
finally:
if worker_stopped:
compose("up", "-d", "--no-deps", "--force-recreate", "worker")
if __name__ == "__main__":
run(main)
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Verify the worker identity is restricted to its required database, S3, and broker actions."""
import json
from pathlib import Path
import sys
import uuid
import pika
import psycopg
from botocore.exceptions import ClientError
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from src.jobs import connect
from src.object_store import ObjectStore
from src.store import database_url
def check_database() -> dict:
with psycopg.connect(database_url()) as connection:
with connection.cursor() as cursor:
cursor.execute("SELECT current_user")
role = cursor.fetchone()[0]
cursor.execute("SELECT id, status FROM attachments LIMIT 1")
cursor.fetchall()
try:
cursor.execute("SELECT id FROM users LIMIT 1")
except psycopg.errors.InsufficientPrivilege:
connection.rollback()
users_denied = True
else:
raise AssertionError("worker can read users")
if role != "messenger_worker":
raise AssertionError(f"unexpected worker database role: {role}")
return {"role": role, "attachments_read": True, "users_read_denied": users_denied}
def check_s3() -> dict:
client = ObjectStore().client
results = {}
for bucket in ("messenger-staging", "messenger-attachments", "messenger-variants"):
try:
client.list_objects_v2(Bucket=bucket, MaxKeys=1)
except ClientError as error:
code = error.response.get("Error", {}).get("Code")
if code not in ("AccessDenied", "403"):
raise
results[f"{bucket}_list_denied"] = True
else:
raise AssertionError(f"worker can list {bucket}")
for bucket, key in (
("messenger-attachments", f"original/{uuid.uuid4()}"),
("messenger-variants", f"variants/{uuid.uuid4()}"),
):
try:
client.get_object(Bucket=bucket, Key=key)
except ClientError as error:
code = error.response.get("Error", {}).get("Code")
if code not in ("AccessDenied", "403"):
raise
results[f"{bucket}_read_denied"] = True
else:
raise AssertionError(f"worker can read objects from {bucket}")
probe = str(uuid.uuid4())
allowed_objects = (
("messenger-staging", f"staging/isolation-probe/{probe}", True),
("messenger-attachments", f"original/isolation-probe/{probe}", False),
("messenger-variants", f"variants/isolation-probe/{probe}", False),
)
for bucket, key, verify_read in allowed_objects:
try:
client.put_object(Bucket=bucket, Key=key, Body=b"worker-scope-probe")
if verify_read:
response = client.get_object(Bucket=bucket, Key=key)
try:
if response["Body"].read() != b"worker-scope-probe":
raise AssertionError("worker staging read returned unexpected data")
finally:
response["Body"].close()
results[f"{bucket}_write"] = True
finally:
try:
client.delete_object(Bucket=bucket, Key=key)
except ClientError as error:
if error.response.get("Error", {}).get("Code") not in ("NoSuchKey", "404"):
raise
results["messenger-staging_read"] = True
results["temporary_objects_removed"] = True
return results
def check_broker() -> dict:
connection = connect()
channel = connection.channel()
try:
try:
channel.basic_publish(
exchange="",
routing_key="attachments.process.v1",
body=b"worker-isolation-probe",
)
channel.close()
except pika.exceptions.ChannelClosedByBroker as error:
if error.reply_code != 403:
raise
publish_denied = True
else:
publish_denied = False
if not publish_denied:
raise AssertionError("worker can publish to the processing queue")
return {"broker_identity_connected": True, "publish_denied": publish_denied}
finally:
if channel.is_open:
channel.close()
if connection.is_open:
connection.close()
def main() -> None:
result = {
"database": check_database(),
"s3": check_s3(),
"broker": check_broker(),
}
print(json.dumps({"result": "PASS", **result}, sort_keys=True))
if __name__ == "__main__":
main()