Compare commits
1
Commits
main
...
lab7-final
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e98b1b5af |
+29
-28
@@ -1,41 +1,42 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Credentials for PostgreSQL/S3/broker are generated by the operator into
|
||||
# ./secrets and injected into OpenBao by scripts/openbao_bootstrap.py.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SERVICE_VERSION=7.0.0
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# lab7: TLS / OpenBao
|
||||
PKI_DIR=/pki
|
||||
OPENBAO_ADDR=https://openbao:8200
|
||||
OPENBAO_CA=/pki/ca.crt
|
||||
OPENBAO_ROLE_ID_FILE=/run/secrets/openbao_role_id
|
||||
OPENBAO_SECRET_ID_FILE=/run/secrets/openbao_secret_id
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_CA=/pki/ca.crt
|
||||
BROKER_CA=/pki/ca.crt
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
LOGIN_RATE_LIMIT=5
|
||||
LOGIN_RATE_WINDOW_SECONDS=60
|
||||
|
||||
# lab5: durable broker + worker (broker credentials live in OpenBao)
|
||||
BROKER_USER=change-me
|
||||
BROKER_PASSWORD=change-me
|
||||
JOB_QUEUE=attachment_jobs
|
||||
STAGING_PREFIX=staging
|
||||
MAX_JOB_ATTEMPTS=5
|
||||
THUMBNAIL_SIZE=256
|
||||
THUMBNAIL_QUALITY=85
|
||||
|
||||
# lab6: observability over TLS
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4317
|
||||
OTEL_CA=/pki/ca.crt
|
||||
OTEL_METRIC_INTERVAL_MS=10000
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
GRAFANA_ADMIN_PASSWORD=change-me
|
||||
|
||||
@@ -1,18 +1,27 @@
|
||||
## Лабораторная 7
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Лесковский Сергей БПИ2304
|
||||
Целевая оценка (3/4/5): 3
|
||||
Трек 5, если заявлен: Не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено:
|
||||
Весь минимум lab6 сохранён, внешний вход переведён на HTTPS (8443, локальный CA с SAN). Добавлено: обязательный mTLS LB→API через TLS-шлюзы с allowlist CN=lb (чужой CA/identity отклоняются), TLS с проверкой сервера для PostgreSQL (verify-full, hostnossl запрещён), MinIO (verify CA) и RabbitMQ (amqps 5671), TLS OTLP в Collector; OpenBao в production-режиме (file storage, init/unseal оператором, policy read-only на kv/messenger, AppRole token TTL 1h/secret_id 24h, без root token в приложении); Argon2id m=19456,t=2,p=1; login rate limit 429+Retry-After; non-root/read_only/cap_drop/no-new-privileges/лимиты для api и worker; заполнена contracts/trust-matrix.md.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone:
|
||||
См. REPORT.md → «Запуск из чистого clone». Кратко: секреты и .env; bash scripts/pki.sh; docker compose up -d openbao; python scripts/openbao_bootstrap.py; docker compose up -d; python tests/smoke.py --base-url https://localhost:8443 --ca-file pki/ca.crt; python scripts/security_internal.py; python scripts/security_rate_limit.py.
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md:
|
||||
Smoke по HTTPS PASS; security.py PASS (чужой CA отклонён); mTLS: без серта 400, чужой CA 400, CN=other 403, CN=lb 200; OpenBao: свой path 200, чужой 403; rate limit 5×401→429 Retry-After 60; persistence PASS; failover PASS (recovery 3.141 s). Доказательства — evidence/smoke.txt, security-external.txt, security-internal.txt, argon2.txt, rate-limit.txt, hardening.txt, persistence.txt, failover-1.txt, traces.json.
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
Один DB/S3/broker identity для api и worker; mTLS к broker/Collector и TLS для telemetry-backends/Grafana→datasources оставлены в доверенной сети (уровень 4); ротация/revocation, rehash, image scan — уровень 4. OpenBao init/unseal — ручной операторский шаг, после рестарта нужен повторный bootstrap. Один Docker-host, HA нет.
|
||||
|
||||
Источники и использование ИИ:
|
||||
OpenBao 2.0.2, nginx 1.27.1, PostgreSQL 16.4, MinIO, RabbitMQ 3.13, OTel Collector 0.115.1, Prometheus/Loki/Tempo/Grafana, FastAPI 0.115, argon2-cffi, pika 1.3.2, boto3 1.35. ИИ использовался для консультаций по AppRole и nginx mTLS.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
@@ -30,3 +30,8 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
# lab7 local PKI material
|
||||
pki/
|
||||
certs/
|
||||
*.srl
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=60 \
|
||||
PIP_RETRIES=2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
# Пробуем несколько зеркал по очереди. Первое, которое ответит, используется.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
set -e; \
|
||||
for idx in \
|
||||
"https://pypi.depkit.ru/simple/" \
|
||||
"https://pypi-mirror.gitverse.ru/simple" \
|
||||
"https://pypi.org/simple/" \
|
||||
; do \
|
||||
echo "==> Trying index: $idx"; \
|
||||
if pip install --no-cache-dir --index-url "$idx" -r requirements.txt; then \
|
||||
echo "==> OK: $idx"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
echo "==> FAILED: $idx"; \
|
||||
done; \
|
||||
echo "==> All indexes failed"; \
|
||||
exit 1
|
||||
|
||||
COPY src ./src
|
||||
|
||||
RUN useradd -r -u 10001 -s /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||
@@ -1,37 +1,69 @@
|
||||
# Отчёт — лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Лесковский Сергей БПИ2304
|
||||
- **Целевая оценка:** 3; трек 5 не заявлен
|
||||
- **Commit/PR:** ветка `lab7-final` форка `Leskovskiy/mtusi2026lab7-Leskovskiy`, PR в `mtusi2026/mtusi2026lab7`
|
||||
- **Использованные источники, библиотеки и помощь ИИ:** OpenBao 2.0.2, nginx 1.27.1 (TLS/mTLS), PostgreSQL 16.4 (TLS), MinIO (TLS), RabbitMQ 3.13 (TLS), OTel Collector 0.115.1 (TLS receiver), Prometheus/Loki/Tempo/Grafana, FastAPI 0.115, Argon2id (argon2-cffi), pika 1.3.2, boto3 1.35. ИИ использовался для консультаций по OpenBao AppRole и конфигурации nginx mTLS.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker Compose v2, Python 3, Git Bash (openssl и обёртки на Windows).
|
||||
|
||||
1. Сгенерировать локальные секреты: `secrets/postgres_password.txt`, `secrets/s3_root_user.txt`, `secrets/s3_root_password.txt`, `secrets/s3_access_key_id.txt`, `secrets/s3_secret_access_key.txt`, `secrets/broker_user.txt`, `secrets/broker_password.txt`; `.env` по `.env.example` (`BROKER_USER`, `BROKER_PASSWORD`, `GRAFANA_ADMIN_PASSWORD`).
|
||||
2. `bash scripts/pki.sh` — локальный CA и leaf-сертификаты в `pki/` (в Git не попадают).
|
||||
3. `docker compose --env-file .env up -d openbao`
|
||||
4. `python scripts/openbao_bootstrap.py` — init (однократно), unseal, KV, policy, AppRole; самопроверка login/policy.
|
||||
5. `docker compose --env-file .env up -d`
|
||||
6. `curl --cacert pki/ca.crt https://localhost:8443/health/ready`
|
||||
7. Тесты: `python tests/smoke.py --base-url https://localhost:8443 --ca-file pki/ca.crt`; `tests/security.py`; `python scripts/security_internal.py`; `python scripts/security_rate_limit.py`.
|
||||
8. Остановка: `docker compose --env-file .env down`; удаление данных: `down -v` (плюс `secrets/openbao_init.json` и `pki/` удаляются вручную).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Клиент --TLS--> lb:8443 --mTLS--> api-gw1/2:8443 --закрытый hop--> api1/api2:8080
|
||||
(CN=lb allowlist) │
|
||||
api/worker --TLS--> PostgreSQL, MinIO, RabbitMQ, OTel Collector, OpenBao
|
||||
```
|
||||
|
||||
- **Внешний TLS:** nginx LB, сертификат с SAN `localhost`/`127.0.0.1`; единственный публичный порт — 8443.
|
||||
- **LB → API:** обязательный mTLS. LB предъявляет сертификат с CN=lb; шлюз проверяет цепочку (`ssl_verify_client on`) и identity (allowlist CN=lb); чужой CA, отсутствие сертификата и CN≠lb отклоняются. Шлюз → API — закрытый compose-hop без публикации портов.
|
||||
- **Хранилища:** PostgreSQL с `ssl=on` и `pg_hba`, запрещающим `hostnossl`; клиент `sslmode=verify-full`. MinIO с TLS (`--certs-dir`), boto3 `verify=CA`. RabbitMQ с отключённым plain-TCP и TLS-listener 5671, pika проверяет hostname.
|
||||
- **Телеметрия:** OTel Collector принимает OTLP по TLS; SDK проверяет CA. Backends (Prometheus/Loki/Tempo/Grafana) — внутри доверенной сети, наружу опубликованы только `127.0.0.1` admin-интерфейсы.
|
||||
- **OpenBao:** production-режим (file storage + TLS), init/unseal оператором, policy `messenger` (read только `kv/data/messenger`), AppRole с token TTL 1h/secret_id TTL 24h. Приложение получает `database_url`, S3-ключи и broker URL через AppRole; root token и unseal key в приложении не используются.
|
||||
- **Argon2id:** `time_cost=2, memory_cost=19456 (19 MiB), parallelism=1, hash_len=32, salt_len=16`; измеренное время verify ~24.5 ms (evidence/argon2.txt).
|
||||
- **Лимиты:** login rate limit (5 попыток/60 с на username+IP, общий через PostgreSQL) → 429 + `Retry-After: 60`; размер запроса ограничен nginx (11 MiB) и API (10 MiB).
|
||||
- **Контейнеры:** api/worker — non-root (`appuser`), `read_only`, `cap_drop: ALL`, `no-new-privileges`, tmpfs `/tmp`, memory 512m, 1 CPU (evidence/hardening.txt).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Предыдущие возможности через HTTPS | `tests/smoke.py --ca-file pki/ca.crt` | `PASS: lab7 public API scenario` | evidence/smoke.txt |
|
||||
| Чужой CA отклоняется | `tests/security.py` | `PASS: trusted HTTPS works; untrusted CA rejected` | evidence/security-external.txt |
|
||||
| LB→API mTLS: нет серта/чужой CA/чужая identity | `scripts/security_internal.py` | 400 / 400 / 403 | evidence/security-internal.txt |
|
||||
| LB identity допускается | `scripts/security_internal.py` | 200 | evidence/security-internal.txt |
|
||||
| OpenBao scoped policy | bootstrap self-check + `security_internal.py` | свой path 200, чужой 403 | evidence/security-internal.txt |
|
||||
| Argon2id параметры | `docker compose exec api1 python …` | `$argon2id$v=19$m=19456,t=2,p=1`, verify ~24.5 ms | evidence/argon2.txt |
|
||||
| Login rate limit | `scripts/security_rate_limit.py` | 5×401, затем 429 + `Retry-After: 60` | evidence/rate-limit.txt |
|
||||
| Хардненинг контейнеров | `docker inspect` | ro, cap_drop ALL, no-new-privileges, non-root, limits | evidence/hardening.txt |
|
||||
| Сохранность и failover через HTTPS | `tests/resilience*.py --ca-file` | persistence PASS; failover PASS (recovery 3.141 s) | evidence/persistence.txt, failover-1.txt |
|
||||
| Телеметрия после TLS | Prometheus/Tempo query | метрики и трассы присутствуют | evidence/traces.json |
|
||||
| Матрица доверия заполнена | `contracts/trust-matrix.md` | все связи, identity, тесты | contracts/trust-matrix.md |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Рестарт stores + OpenBao + API (persistence hook) | Сессии/данные сохранены, OpenBao разsealed оператором | `PASS: persistence preserved...` | потерь нет |
|
||||
| Остановка одной API-реплики | LB уходит на вторую, mTLS сохраняется | 12 записей, recovery 3.141 s, 9 неудачных попыток в окне | потерь подтверждённых сообщений нет |
|
||||
| Остановка/рестарт OpenBao | API не стартует, пока OpenBao sealed; после unseal восстанавливается | bootstrap → API healthy | простой на время unseal |
|
||||
| Недоверенный клиентский сертификат | TLS/HTTP отказ | 400/403 | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Раздельные DB/S3/broker identity для api и worker не реализованы (один пользователь на сервис); mTLS к broker/Collector, TLS для telemetry-backends и Grafana→datasources оставлены внутри доверенной сети — это уровень 4. Для продакшена нужны отдельные роли, scoped S3-политики и TLS/mTLS на всех внутренних hop.
|
||||
- Ротация/expiry/revocation короткоживущих сертификатов и SecretID, rehash пароля, dependency/image scan, read-only для stateful-сервисов — уровень 4.
|
||||
- OpenBao init/unseal — ручной операторский шаг (разрешённый компромисс); после рестарта требуется повторный bootstrap.
|
||||
- Один Docker-host; HA хранилищ и OpenBao не заявляется.
|
||||
- Argon2id параметры обоснованы измерением на этом стенде (~24.5 ms verify), memory 19 MiB соответствует рекомендации OWASP для интерактивного логина.
|
||||
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
name: lab7
|
||||
|
||||
x-api-env: &api_env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SERVICE_VERSION: "7.0.0"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
PKI_DIR: /pki
|
||||
OPENBAO_ADDR: https://openbao:8200
|
||||
OPENBAO_CA: /pki/ca.crt
|
||||
OPENBAO_ROLE_ID_FILE: /run/secrets/openbao_role_id
|
||||
OPENBAO_SECRET_ID_FILE: /run/secrets/openbao_secret_id
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_CA: /pki/ca.crt
|
||||
BROKER_CA: /pki/ca.crt
|
||||
JOB_QUEUE: attachment_jobs
|
||||
STAGING_PREFIX: staging
|
||||
MAX_JOB_ATTEMPTS: "5"
|
||||
THUMBNAIL_SIZE: "256"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
LOGIN_RATE_LIMIT: "5"
|
||||
LOGIN_RATE_WINDOW_SECONDS: "60"
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4317
|
||||
OTEL_CA: /pki/ca.crt
|
||||
OTEL_METRIC_INTERVAL_MS: "10000"
|
||||
|
||||
x-hardening: &hardening
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
install -m 0600 -o postgres /pki/postgres.key /tmp/postgres.key &&
|
||||
install -m 0644 /pki/postgres.crt /tmp/postgres.crt &&
|
||||
install -m 0644 /pki/ca.crt /tmp/ca.crt &&
|
||||
exec docker-entrypoint.sh postgres
|
||||
-c ssl=on
|
||||
-c ssl_cert_file=/tmp/postgres.crt
|
||||
-c ssl_key_file=/tmp/postgres.key
|
||||
-c ssl_ca_file=/tmp/ca.crt
|
||||
-c hba_file=/etc/postgresql/pg_hba.conf
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
- ./pki:/pki:ro
|
||||
secrets:
|
||||
- postgres_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: milvusdb/minio:RELEASE.2024-12-18T13-15-44Z
|
||||
command: server /data --certs-dir /certs --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
volumes:
|
||||
- s3data:/data
|
||||
- ./pki:/pki:ro
|
||||
- ./pki/minio.crt:/certs/public.crt:ro
|
||||
- ./pki/minio.key:/certs/private.key:ro
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS --cacert /pki/ca.crt https://localhost:9000/minio/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:?set BROKER_USER in .env}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?set BROKER_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitdata:/var/lib/rabbitmq
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
|
||||
- ./pki:/pki:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
openbao-perms:
|
||||
image: openbao/openbao:2.0.2
|
||||
user: "0:0"
|
||||
entrypoint: ["sh", "-c", "chown -R 100:1000 /openbao/data"]
|
||||
volumes:
|
||||
- openbaodata:/openbao/data
|
||||
|
||||
openbao:
|
||||
image: openbao/openbao:2.0.2
|
||||
command: ["server", "-config=/openbao/config.hcl"]
|
||||
environment:
|
||||
BAO_ADDR: https://openbao:8200
|
||||
BAO_CACERT: /pki/ca.crt
|
||||
cap_add:
|
||||
- IPC_LOCK
|
||||
ports:
|
||||
- "127.0.0.1:8200:8200"
|
||||
volumes:
|
||||
- openbaodata:/openbao/data
|
||||
- ./infra/openbao/config.hcl:/openbao/config.hcl:ro
|
||||
- ./pki:/pki:ro
|
||||
depends_on:
|
||||
openbao-perms:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "BAO_ADDR=https://openbao:8200 BAO_CACERT=/pki/ca.crt bao status >/dev/null 2>&1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
api1:
|
||||
build: .
|
||||
image: lab7-api:local
|
||||
<<: *hardening
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-1
|
||||
secrets:
|
||||
- openbao_role_id
|
||||
- openbao_secret_id
|
||||
volumes:
|
||||
- ./pki:/pki:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
openbao:
|
||||
condition: service_healthy
|
||||
|
||||
api2:
|
||||
image: lab7-api:local
|
||||
<<: *hardening
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-2
|
||||
secrets:
|
||||
- openbao_role_id
|
||||
- openbao_secret_id
|
||||
volumes:
|
||||
- ./pki:/pki:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
openbao:
|
||||
condition: service_healthy
|
||||
|
||||
worker:
|
||||
image: lab7-api:local
|
||||
<<: *hardening
|
||||
command: ["python", "-m", "src.worker"]
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: worker
|
||||
secrets:
|
||||
- openbao_role_id
|
||||
- openbao_secret_id
|
||||
volumes:
|
||||
- ./pki:/pki:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
openbao:
|
||||
condition: service_healthy
|
||||
|
||||
api-gw1:
|
||||
image: nginx:1.27.1-alpine
|
||||
environment:
|
||||
API_UPSTREAM: api1
|
||||
volumes:
|
||||
- ./infra/nginx/gateway.conf.template:/etc/nginx/templates/gateway.conf.template:ro
|
||||
- ./pki:/pki:ro
|
||||
ports:
|
||||
- "127.0.0.1:9443:8443"
|
||||
depends_on:
|
||||
- api1
|
||||
|
||||
api-gw2:
|
||||
image: nginx:1.27.1-alpine
|
||||
environment:
|
||||
API_UPSTREAM: api2
|
||||
volumes:
|
||||
- ./infra/nginx/gateway.conf.template:/etc/nginx/templates/gateway.conf.template:ro
|
||||
- ./pki:/pki:ro
|
||||
ports:
|
||||
- "127.0.0.1:9444:8443"
|
||||
depends_on:
|
||||
- api2
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.1-alpine
|
||||
ports:
|
||||
- "8443:8443"
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./pki:/pki:ro
|
||||
depends_on:
|
||||
- api-gw1
|
||||
- api-gw2
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.115.1
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
- ./pki:/pki:ro
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- promdata:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
depends_on:
|
||||
- otel-collector
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.2.1
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.3.1
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
|
||||
GF_USERS_DEFAULT_THEME: dark
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
rabbitdata: {}
|
||||
openbaodata: {}
|
||||
promdata: {}
|
||||
lokidata: {}
|
||||
tempodata: {}
|
||||
grafanadata: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password.txt
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user.txt
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password.txt
|
||||
openbao_role_id:
|
||||
file: ./secrets/openbao_role_id.txt
|
||||
openbao_secret_id:
|
||||
file: ./secrets/openbao_secret_id.txt
|
||||
+25
-29
@@ -1,37 +1,33 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия — lab7 (реализация)
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Локальный CA `pki/ca.crt` создаётся оператором через `scripts/pki.sh` (не коммитится). Каждый leaf имеет CN и SAN; серверы проверяют цепочку и hostname, mTLS-шлюзы дополнительно проверяют identity клиента.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
| Связь | Реализация (порт/endpoint) | CA / identity / проверка | Срок credential | Тест |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | `https://localhost:8443` (nginx, TLS 1.2/1.3) | `ca.crt`, SAN `DNS:localhost,IP:127.0.0.1`; hostname проверяется клиентом; Bearer-сессия | сертификат 825 дней; сессия `SESSION_TTL_SECONDS` | `tests/security.py` (чужой CA → ошибка верификации); `tests/smoke.py` |
|
||||
| LB → API (все реплики) | LB → `api-gw1/api-gw2:8443` (mTLS), шлюз → API `:8080` по закрытому compose-hop | серверный серт `gateway` (SAN api-gw1/api-gw2), клиентский серт `lb` (CN=lb); `ssl_verify_client on`, allowlist `CN=lb`, чужой CA/identity отклоняются | leaf 825 дней; CN allowlist | `scripts/security_internal.py`: без серта → 400, чужой CA → 400, CN=other → 403, CN=lb → 200 |
|
||||
| API/worker → PostgreSQL | `postgres:5432`, `sslmode=verify-full` + `sslrootcert=/pki/ca.crt` | CA + SAN `postgres`; `pg_hba` запрещает `hostnossl`; роль `messenger` с правами только на свою БД | пароль из OpenBao | smoke + persistence (подключение падает при неверном CA) |
|
||||
| API/worker → S3 (MinIO) | `https://s3:9000`, boto3 `verify=/pki/ca.crt` | CA + SAN `s3`; bucket приватный; ключи из OpenBao | ключи из OpenBao | smoke (upload/download) |
|
||||
| API/worker → broker | `amqps://broker:5671` (TLS listener, plain TCP listener отключён) | CA + SAN `broker`; pika `SSLOptions` с проверкой hostname | пароль из OpenBao | smoke (job → ready), failover |
|
||||
| API/worker → Collector | `https://otel-collector:4317` (OTLP/gRPC + TLS) | CA + SAN `otel-collector`; gRPC `ssl_channel_credentials` | — | метрики/трассы/логи в Grafana |
|
||||
| Collector → telemetry backends | внутри доверенной сети compose (Tempo/Loki/Prometheus) | **исключение**: сеть стенда считается доверенной зоной; публичных портов у backends нет | — | отчёт, ограничения |
|
||||
| Grafana → data sources | внутри доверенной сети, порт `127.0.0.1:3000` | provisioning datasources; наружу не публикуется | — | Grafana health |
|
||||
| API/worker → OpenBao | `https://openbao:8200`, AppRole login → token, KV v2 read `kv/data/messenger` | CA + SAN `openbao`; policy `messenger` разрешает только чтение своего path; root token приложению недоступен | SecretID 24h, token TTL 1h, max 4h | `scripts/security_internal.py`: свой path 200, чужой 403; bootstrap self-check |
|
||||
| Оператор → OpenBao/Grafana/Prometheus/S3 admin | `127.0.0.1:8200/3000/9090`, admin API MinIO не публикуется | OpenBao TLS; root token/unseal key только в `secrets/openbao_init.json` у оператора | — | bootstrap, отчёт |
|
||||
| API → Redis/Valkey | N/A | не используется (сессии в PostgreSQL) | — | — |
|
||||
| Replication/кластерные каналы | N/A | одиночные узлы | — | — |
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
1. HTTPS с правильным CA/hostname проходит; с чужим CA и неверным hostname — ошибка верификации: `tests/security.py` PASS.
|
||||
2. LB→API: без клиентского серта / чужой CA / запрещённая identity → отказ; CN=lb → успех: `scripts/security_internal.py` PASS.
|
||||
3. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; `X-User-Id` не подменяет Bearer: публичный smoke PASS.
|
||||
4. OpenBao app policy читает только `kv/messenger`, запись и чужой path запрещены: self-check и `scripts/security_internal.py` PASS.
|
||||
5. Login rate limit: 5 неудач → 429 + `Retry-After`: `scripts/security_rate_limit.py` PASS.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
- `scripts/pki.sh` создаёт CA и leaf-сертификаты в `pki/` (gitignored). CA key и leaf keys остаются у оператора.
|
||||
- `scripts/openbao_bootstrap.py` (после `docker compose up -d openbao`) выполняет init (однократно), unseal, включение `kv` v2, запись runtime-секретов, policy, AppRole; root token и unseal key сохраняет в `secrets/openbao_init.json` (gitignored).
|
||||
- После рестарта OpenBao он снова sealed: оператор повторяет bootstrap (идемпотентно). Это же делает `scripts/restart.sh` в persistence-сценарии.
|
||||
- Приложение получает секреты только через AppRole; root token, unseal keys и SecretID не попадают в образы, Git и телеметрию.
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,16 @@
|
||||
SERVICE STATUS PORTS
|
||||
api-gw1 Up About a minute 127.0.0.1:9443->8443/tcp
|
||||
api-gw2 Up About a minute 127.0.0.1:9444->8443/tcp
|
||||
api1 Up About a minute 8080/tcp
|
||||
api2 Up About a minute 8080/tcp
|
||||
broker Up 3 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
grafana Up 6 minutes 127.0.0.1:3000->3000/tcp
|
||||
lb Up About a minute 0.0.0.0:8443->8443/tcp, [::]:8443->8443/tcp
|
||||
loki Up 6 minutes 3100/tcp
|
||||
openbao Up 3 minutes (healthy) 127.0.0.1:8200->8200/tcp
|
||||
otel-collector Up 6 minutes 4317-4318/tcp, 55678-55679/tcp
|
||||
postgres Up 3 minutes (healthy) 5432/tcp
|
||||
prometheus Up 6 minutes 127.0.0.1:9090->9090/tcp
|
||||
s3 Up 3 minutes (healthy) 9000/tcp
|
||||
tempo Up 6 minutes
|
||||
worker Up 3 minutes 8080/tcp
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"traces":[{"traceID":"10e1b3be92e1a99f516fb72cb89d5da8","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/messages","startTimeUnixNano":"1795110028472495804","durationMs":8},{"traceID":"1c58ef73eeca445458f3d8b66f0c7669","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats","startTimeUnixNano":"1795110028052560529","durationMs":22}],"metrics":{"inspectedTraces":38,"inspectedBytes":"20488","completedJobs":1,"totalJobs":1}}
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger — lab6 observability",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"editable": true,
|
||||
"time": { "from": "now-30m", "to": "now" },
|
||||
"timezone": "utc",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API RPS (excluding health)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m]))",
|
||||
"legendFormat": "RPS"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API 5xx ratio",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{http_response_status_code=~\"5..\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])), 0.001)",
|
||||
"legendFormat": "5xx / business"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP duration p95",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Attachment jobs by outcome",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (job_outcome) (rate(attachment_jobs_total[1m]))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker job duration p95 by outcome",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, job_outcome) (rate(attachment_job_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "API request rate by route (top 10)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "topk(10, sum by (http_route, http_request_method) (rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: "-1m"
|
||||
spanEndTimeShift: "1m"
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
@@ -0,0 +1,26 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
@@ -0,0 +1,30 @@
|
||||
map $ssl_client_s_dn $lb_identity_ok {
|
||||
default 0;
|
||||
"~*CN=lb(,|/|$)" 1;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8443 ssl;
|
||||
server_name api-gateway;
|
||||
ssl_certificate /pki/gateway.crt;
|
||||
ssl_certificate_key /pki/gateway.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_client_certificate /pki/ca.crt;
|
||||
ssl_verify_client on;
|
||||
ssl_verify_depth 2;
|
||||
|
||||
# A trusted CA is not enough: only the LB identity may call the API.
|
||||
if ($lb_identity_ok = 0) {
|
||||
return 403;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://${API_UPSTREAM}:8080;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Client-Identity $ssl_client_s_dn;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
events {}
|
||||
|
||||
http {
|
||||
upstream api_secure {
|
||||
server api-gw1:8443 max_fails=1 fail_timeout=2s;
|
||||
server api-gw2:8443 max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8443 ssl;
|
||||
server_name localhost;
|
||||
ssl_certificate /pki/server.crt;
|
||||
ssl_certificate_key /pki/server.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_session_cache shared:SSL:5m;
|
||||
client_max_body_size 11m;
|
||||
|
||||
location / {
|
||||
proxy_pass https://api_secure;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 3s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
# A stopped replica makes its gateway answer 502; retry the other
|
||||
# gateway. Message POSTs carry Idempotency-Key, so a retry is safe.
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 3s;
|
||||
# Mandatory mTLS to every API replica: verify the gateway chain and
|
||||
# present the LB identity certificate.
|
||||
proxy_ssl_verify on;
|
||||
proxy_ssl_trusted_certificate /pki/ca.crt;
|
||||
proxy_ssl_certificate /pki/lb.crt;
|
||||
proxy_ssl_certificate_key /pki/lb.key;
|
||||
proxy_ssl_server_name on;
|
||||
proxy_ssl_name api-gw1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
ui = false
|
||||
disable_mlock = true
|
||||
|
||||
storage "file" {
|
||||
path = "/openbao/data"
|
||||
}
|
||||
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_disable = 0
|
||||
tls_cert_file = "/pki/openbao.crt"
|
||||
tls_key_file = "/pki/openbao.key"
|
||||
tls_min_version = "tls12"
|
||||
}
|
||||
|
||||
api_addr = "https://openbao:8200"
|
||||
cluster_addr = "https://openbao:8201"
|
||||
@@ -0,0 +1,8 @@
|
||||
# Application policy: read only the KV v2 data and metadata of its own path.
|
||||
path "kv/data/messenger" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "kv/metadata/messenger" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
tls:
|
||||
cert_file: /pki/otel.crt
|
||||
key_file: /pki/otel.key
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /pki/otel.crt
|
||||
key_file: /pki/otel.key
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 256
|
||||
spike_limit_mib: 64
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: false
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
loki:
|
||||
endpoint: http://loki:3100/loki/api/v1/push
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [loki]
|
||||
@@ -0,0 +1,4 @@
|
||||
# Require TLS for every TCP connection; no plaintext host entry is accepted.
|
||||
local all all trust
|
||||
hostssl all all all scram-sha-256
|
||||
hostnossl all all all reject
|
||||
@@ -0,0 +1,8 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
@@ -0,0 +1,10 @@
|
||||
listeners.tcp = none
|
||||
listeners.ssl.default = 5671
|
||||
|
||||
ssl_options.cacertfile = /pki/ca.crt
|
||||
ssl_options.certfile = /pki/broker.crt
|
||||
ssl_options.keyfile = /pki/broker.key
|
||||
ssl_options.verify = verify_none
|
||||
ssl_options.fail_if_no_peer_cert = false
|
||||
|
||||
loopback_users = none
|
||||
@@ -0,0 +1,19 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
@@ -0,0 +1,12 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
sqlalchemy==2.0.35
|
||||
psycopg[binary]==3.2.3
|
||||
boto3==1.35.36
|
||||
argon2-cffi==23.1.0
|
||||
pillow==10.4.0
|
||||
pydantic==2.9.2
|
||||
pika==1.3.2
|
||||
opentelemetry-sdk==1.28.2
|
||||
opentelemetry-exporter-otlp==1.28.2
|
||||
opentelemetry-instrumentation-fastapi==0.49b2
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab5 failure drills: broker restart durability and bounded retry.
|
||||
|
||||
A) Stop the worker, accept an upload, restart the broker (persistent volume),
|
||||
start the worker: the accepted job still becomes ready.
|
||||
B) Accept an upload while the worker is down, stop S3, start the worker:
|
||||
transient failures exhaust MAX_JOB_ATTEMPTS and the attachment ends as
|
||||
failed with a safe error_code. S3 is restored afterwards.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_status(c, actor, att, wanted, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
row = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
|
||||
if row["status"] in wanted:
|
||||
return row
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("attachment did not reach " + repr(wanted))
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 failure drills")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("A. broker restart keeps accepted work")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("restart", "broker")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover after broker restart")
|
||||
time.sleep(0.5)
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
print(json.dumps({
|
||||
"scenario": "broker_restart",
|
||||
"attachment_id": att["id"],
|
||||
"status": ready["status"],
|
||||
"seconds_to_ready": round(time.monotonic() - started, 3),
|
||||
}, ensure_ascii=False))
|
||||
check(ready["status"] == "ready", "job lost after broker restart")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
print("B. bounded retry ends in a safe failed state")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("stop", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
failed = wait_status(c, bob, att, {"failed"}, seconds=180)
|
||||
print(json.dumps({
|
||||
"scenario": "retry_exhausted",
|
||||
"attachment_id": att["id"],
|
||||
"status": failed["status"],
|
||||
"error_code": failed["error_code"],
|
||||
}, ensure_ascii=False))
|
||||
check(failed["error_code"] in ("processing_failed", "invalid_image"), "unsafe error_code")
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
finally:
|
||||
compose("up", "-d", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "S3 did not recover")
|
||||
time.sleep(0.5)
|
||||
print("PASS: broker durability and bounded retry demonstrated.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Demonstrate the lab5 acceptance scenario without hiding infrastructure steps.
|
||||
|
||||
Worker stopped -> upload returns 202 queued -> message references the chat
|
||||
attachment -> API replicas restarted -> worker started -> the same job
|
||||
reaches ready with a contract-compliant JPEG thumbnail.
|
||||
|
||||
The script prints only synthetic IDs and timings. It never prints tokens.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 worker-outage demonstration")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("1. stop worker")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
print("2. upload with worker down")
|
||||
att, content = c.attachment(alice, chat)
|
||||
check(att["status"] == "queued", "queued expected after upload")
|
||||
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
|
||||
check(meta["status"] in ("queued", "processing"), "still unprocessed while worker is down")
|
||||
pending = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
check(pending["status"] == 409, "content must be 409 while queued")
|
||||
|
||||
print("3. attach the queued attachment to a message")
|
||||
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(msg["attachment_ids"] == [att["id"]], "message lost the attachment")
|
||||
|
||||
print("4. restart API replicas (accepted work must survive)")
|
||||
compose("restart", "api1", "api2")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover")
|
||||
time.sleep(0.5)
|
||||
|
||||
print("5. start worker")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
check(ready["status"] == "ready", "job did not reach ready")
|
||||
variants = [v for v in ready["variants"] if v["name"] == "thumbnail"]
|
||||
check(len(variants) == 1, "expected exactly one thumbnail")
|
||||
v = variants[0]
|
||||
check((v["width"], v["height"]) == (256, 160), "thumbnail dimensions differ")
|
||||
thumb = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob)
|
||||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be JPEG")
|
||||
from image_probe import jpeg_size
|
||||
check(jpeg_size(thumb["bytes"]) == (256, 160), "actual JPEG size differs")
|
||||
|
||||
print(json.dumps({
|
||||
"attachment_id": att["id"],
|
||||
"message_id": msg["id"],
|
||||
"status_after_worker_restart": ready["status"],
|
||||
"thumbnail": {"width": v["width"], "height": v["height"],
|
||||
"size_bytes": v["size_bytes"], "sha256": v["sha256"]},
|
||||
"seconds_from_worker_start_to_ready": elapsed,
|
||||
}, ensure_ascii=False))
|
||||
print("PASS: queued job completed after API and worker restart.")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator bootstrap for OpenBao (lab7).
|
||||
|
||||
Run AFTER `docker compose up -d openbao` and BEFORE starting the API:
|
||||
|
||||
python scripts/openbao_bootstrap.py
|
||||
|
||||
It initializes OpenBao (once), unseals it, creates the KV v2 path, writes the
|
||||
runtime credentials that are already in ./secrets, installs the scoped policy,
|
||||
enables AppRole and writes role_id/secret_id files for the application.
|
||||
|
||||
Operator-held material (unseal key, root token) is written to
|
||||
./secrets/openbao_init.json and never leaves this machine. The application
|
||||
never sees it. Delete ./secrets/openbao_init.json only after moving unseal
|
||||
material to a safe place.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ENV = ["docker", "compose", "--env-file", ".env"]
|
||||
INIT_FILE = ROOT / "secrets" / "openbao_init.json"
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (ROOT / "secrets" / name).read_text().strip()
|
||||
|
||||
|
||||
def bao(*args: str, stdin: str | None = None, token: str | None = None):
|
||||
command = ENV + ["exec", "-T"]
|
||||
if token:
|
||||
command += ["-e", f"BAO_TOKEN={token}"]
|
||||
command += ["openbao", "bao", *args]
|
||||
return subprocess.run(command, cwd=ROOT, input=stdin, text=True, capture_output=True)
|
||||
|
||||
|
||||
def wait_for_container() -> None:
|
||||
for _ in range(60):
|
||||
probe = bao("status")
|
||||
if probe.returncode in (0, 2):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit("OpenBao container is not reachable: " + probe.stderr.strip())
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
probe = bao("status", "-format=json")
|
||||
try:
|
||||
return json.loads(probe.stdout)
|
||||
except ValueError:
|
||||
return {"initialized": False, "sealed": True}
|
||||
|
||||
|
||||
def ensure_initialized() -> tuple[str, str]:
|
||||
current = status()
|
||||
if not current.get("initialized", False):
|
||||
if INIT_FILE.exists():
|
||||
raise SystemExit("OpenBao is uninitialized but secrets/openbao_init.json exists; "
|
||||
"refusing to overwrite operator material")
|
||||
init = bao("operator", "init", "-format=json", "-key-shares=1", "-key-threshold=1")
|
||||
if init.returncode != 0:
|
||||
raise SystemExit("operator init failed: " + init.stderr.strip())
|
||||
data = json.loads(init.stdout)
|
||||
INIT_FILE.write_text(json.dumps(data, indent=2))
|
||||
INIT_FILE.chmod(0o600)
|
||||
print("OpenBao initialized; unseal key and root token saved to secrets/openbao_init.json")
|
||||
data = json.loads(INIT_FILE.read_text())
|
||||
key = data["unseal_keys_b64"][0]
|
||||
token = data["root_token"]
|
||||
return key, token
|
||||
|
||||
|
||||
def ensure_unsealed(key: str) -> None:
|
||||
if status().get("sealed", True):
|
||||
unseal = bao("operator", "unseal", key)
|
||||
if unseal.returncode != 0:
|
||||
raise SystemExit("unseal failed: " + unseal.stderr.strip())
|
||||
print("OpenBao unsealed")
|
||||
|
||||
|
||||
def ensure_kv(token: str) -> None:
|
||||
mounts = bao("secrets", "list", "-format=json", token=token)
|
||||
existing = json.loads(mounts.stdout or "{}")
|
||||
if "kv/" not in existing:
|
||||
enable = bao("secrets", "enable", "-path=kv", "-version=2", "kv", token=token)
|
||||
if enable.returncode != 0:
|
||||
raise SystemExit("kv enable failed: " + enable.stderr.strip())
|
||||
print("Enabled kv v2 at kv/")
|
||||
|
||||
|
||||
def write_runtime_secrets(token: str) -> None:
|
||||
pg_password = read("postgres_password.txt")
|
||||
broker_user = read("broker_user.txt")
|
||||
broker_password = read("broker_password.txt")
|
||||
database_url = ("postgresql+psycopg://messenger:" + pg_password +
|
||||
"@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt")
|
||||
broker_url = f"amqps://{broker_user}:{broker_password}@broker:5671/"
|
||||
write = bao("kv", "put", "kv/messenger",
|
||||
f"database_url={database_url}",
|
||||
f"s3_access_key_id={read('s3_access_key_id.txt')}",
|
||||
f"s3_secret_access_key={read('s3_secret_access_key.txt')}",
|
||||
f"broker_url={broker_url}",
|
||||
token=token)
|
||||
if write.returncode != 0:
|
||||
raise SystemExit("kv put failed: " + write.stderr.strip())
|
||||
print("Wrote kv/messenger")
|
||||
|
||||
|
||||
def install_policy(token: str) -> None:
|
||||
policy = (ROOT / "infra" / "openbao" / "policy-messenger.hcl").read_text()
|
||||
write = bao("policy", "write", "messenger", "-", stdin=policy, token=token)
|
||||
if write.returncode != 0:
|
||||
raise SystemExit("policy write failed: " + write.stderr.strip())
|
||||
print("Installed policy messenger")
|
||||
|
||||
|
||||
def ensure_approle(token: str) -> None:
|
||||
enable = bao("auth", "enable", "approle", token=token)
|
||||
if enable.returncode != 0 and "path is already in use" not in (enable.stderr or ""):
|
||||
if "already enabled" not in (enable.stderr or ""):
|
||||
raise SystemExit("approle enable failed: " + enable.stderr.strip())
|
||||
role = bao("write", "auth/approle/role/messenger",
|
||||
"token_ttl=1h", "token_max_ttl=4h", "secret_id_ttl=24h",
|
||||
"policies=messenger", token=token)
|
||||
if role.returncode != 0:
|
||||
raise SystemExit("approle role failed: " + role.stderr.strip())
|
||||
role_id = bao("read", "-field=role_id", "auth/approle/role/messenger/role-id", token=token)
|
||||
secret_id = bao("write", "-f", "-field=secret_id", "auth/approle/role/messenger/secret-id", token=token)
|
||||
if role_id.returncode != 0 or secret_id.returncode != 0:
|
||||
raise SystemExit("could not issue AppRole credentials")
|
||||
(ROOT / "secrets" / "openbao_role_id.txt").write_text(role_id.stdout.strip())
|
||||
(ROOT / "secrets" / "openbao_secret_id.txt").write_text(secret_id.stdout.strip())
|
||||
print("Issued AppRole role_id/secret_id")
|
||||
|
||||
|
||||
def self_check(role_id: str, secret_id: str) -> None:
|
||||
login = bao("write", "-format=json", "auth/approle/login",
|
||||
f"role_id={role_id}", f"secret_id={secret_id}")
|
||||
if login.returncode != 0:
|
||||
raise SystemExit("AppRole login self-check failed: " + login.stderr.strip())
|
||||
token = json.loads(login.stdout)["auth"]["client_token"]
|
||||
read_back = bao("kv", "get", "-format=json", "kv/messenger", token=token)
|
||||
if read_back.returncode != 0:
|
||||
raise SystemExit("policy cannot read kv/messenger: " + read_back.stderr.strip())
|
||||
forbidden = bao("kv", "get", "-format=json", "kv/other", token=token)
|
||||
if forbidden.returncode == 0:
|
||||
raise SystemExit("policy unexpectedly reads kv/other")
|
||||
print("AppRole login and scoped policy verified (kv/other denied)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
wait_for_container()
|
||||
key, root = ensure_initialized()
|
||||
ensure_unsealed(key)
|
||||
ensure_kv(root)
|
||||
write_runtime_secrets(root)
|
||||
install_policy(root)
|
||||
ensure_approle(root)
|
||||
self_check((ROOT / "secrets" / "openbao_role_id.txt").read_text().strip(),
|
||||
(ROOT / "secrets" / "openbao_secret_id.txt").read_text().strip())
|
||||
print("OpenBao bootstrap complete. Start the rest: docker compose up -d")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/bin/sh
|
||||
# Local lab7 PKI bootstrap. Never commit pki/: the CA key and leaf keys are
|
||||
# operator-held material. Run from the repository root (Git Bash on Windows).
|
||||
set -eu
|
||||
|
||||
# Git Bash on Windows rewrites arguments that look like Unix paths (e.g. the
|
||||
# openssl -subj value). Keep the arguments byte-exact.
|
||||
export MSYS_NO_PATHCONV=1
|
||||
|
||||
DAYS=825
|
||||
PKI=${PKI_DIR:-pki}
|
||||
mkdir -p "$PKI"
|
||||
cd "$PKI"
|
||||
|
||||
if [ -f ca.crt ]; then
|
||||
echo "PKI already exists in $PKI; delete it to regenerate."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
write_ext() {
|
||||
# $1 file, $2 san
|
||||
cat > "$1" <<EOF
|
||||
basicConstraints=CA:FALSE
|
||||
keyUsage=digitalSignature,keyEncipherment
|
||||
extendedKeyUsage=serverAuth,clientAuth
|
||||
subjectAltName=$2
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "==> CA"
|
||||
openssl genrsa -out ca.key 4096 >/dev/null 2>&1
|
||||
openssl req -x509 -new -key ca.key -sha256 -days $DAYS \
|
||||
-subj "/CN=mtusi-lab7-local-ca" \
|
||||
-addext "basicConstraints=critical,CA:TRUE" \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-out ca.crt >/dev/null 2>&1
|
||||
|
||||
issue() {
|
||||
# $1 name, $2 CN, $3 SAN
|
||||
openssl genrsa -out "$1.key" 2048 >/dev/null 2>&1
|
||||
openssl req -new -key "$1.key" -subj "/CN=$2" -out "$1.csr" >/dev/null 2>&1
|
||||
write_ext "$1.ext" "$3"
|
||||
openssl x509 -req -in "$1.csr" -CA ca.crt -CAkey ca.key -CAcreateserial \
|
||||
-days $DAYS -sha256 -extfile "$1.ext" -out "$1.crt" >/dev/null 2>&1
|
||||
rm -f "$1.csr" "$1.ext"
|
||||
chmod 600 "$1.key" 2>/dev/null || true
|
||||
}
|
||||
|
||||
echo "==> leaf certificates"
|
||||
issue server "localhost" "DNS:localhost,IP:127.0.0.1"
|
||||
issue lb "lb" "DNS:lb"
|
||||
issue gateway "api-gateway" "DNS:api-gw1,DNS:api-gw2,DNS:localhost"
|
||||
issue postgres "postgres" "DNS:postgres"
|
||||
issue minio "s3" "DNS:s3,DNS:minio,DNS:localhost"
|
||||
issue broker "broker" "DNS:broker"
|
||||
issue otel "otel-collector" "DNS:otel-collector"
|
||||
issue openbao "openbao" "DNS:openbao,IP:127.0.0.1"
|
||||
issue other "other-service" "DNS:other-service"
|
||||
|
||||
# A second, untrusted CA and a client certificate signed by it: used by the
|
||||
# negative mTLS test (valid chain, but the wrong root).
|
||||
echo "==> foreign CA (negative test only)"
|
||||
openssl genrsa -out foreign-ca.key 2048 >/dev/null 2>&1
|
||||
openssl req -x509 -new -key foreign-ca.key -sha256 -days $DAYS \
|
||||
-subj "/CN=foreign-ca" \
|
||||
-addext "basicConstraints=critical,CA:TRUE" \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-out foreign-ca.crt >/dev/null 2>&1
|
||||
openssl genrsa -out foreign-client.key 2048 >/dev/null 2>&1
|
||||
openssl req -new -key foreign-client.key -subj "/CN=foreign-client" -out foreign-client.csr >/dev/null 2>&1
|
||||
write_ext foreign-client.ext "DNS:foreign-client"
|
||||
openssl x509 -req -in foreign-client.csr -CA foreign-ca.crt -CAkey foreign-ca.key \
|
||||
-CAcreateserial -days $DAYS -sha256 -extfile foreign-client.ext -out foreign-client.crt >/dev/null 2>&1
|
||||
rm -f foreign-client.csr foreign-client.ext
|
||||
|
||||
rm -f ca.srl foreign-ca.srl
|
||||
echo "PKI ready in $PKI/"
|
||||
ls -1 *.crt
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0pki.sh"
|
||||
Executable → Regular
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart the stores and the OpenBao node. OpenBao comes back sealed, so the
|
||||
# documented operator step is to re-run the bootstrap (idempotent unseal)
|
||||
# before the API replicas need their secrets again.
|
||||
docker compose --env-file .env restart postgres s3 broker openbao
|
||||
if command -v py >/dev/null 2>&1; then
|
||||
py -3 scripts/openbao_bootstrap.py
|
||||
else
|
||||
python scripts/openbao_bootstrap.py
|
||||
fi
|
||||
docker compose --env-file .env restart api1 api2 api-gw1 api-gw2 worker lb
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0restart.sh"
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative/positive checks for the lab7 trust matrix.
|
||||
|
||||
Covers:
|
||||
* LB -> API mTLS: no client cert, foreign CA, trusted CA but wrong identity,
|
||||
correct LB identity;
|
||||
* external HTTPS: untrusted CA is rejected (make test-security also covers it);
|
||||
* OpenBao: the application policy reads only kv/messenger and is denied
|
||||
elsewhere; the AppRole login works with the issued credentials.
|
||||
|
||||
Run from the repository root after the stand is up:
|
||||
python scripts/security_internal.py
|
||||
"""
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKI = ROOT / "pki"
|
||||
CA = str(PKI / "ca.crt")
|
||||
GATEWAY_URL = "https://localhost:9443/health/live"
|
||||
OPENBAO = "https://127.0.0.1:8200"
|
||||
|
||||
|
||||
def client_context(cert: str | None, key: str | None = None) -> ssl.SSLContext:
|
||||
ctx = ssl.create_default_context(cafile=CA)
|
||||
ctx.check_hostname = True
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
if cert:
|
||||
ctx.load_cert_chain(certfile=str(PKI / cert), keyfile=str(PKI / (key or cert.replace(".crt", ".key"))))
|
||||
return ctx
|
||||
|
||||
|
||||
def get(url: str, ctx: ssl.SSLContext, token: str | None = None):
|
||||
headers = {"X-Vault-Token": token} if token else {}
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=ctx, timeout=10) as response:
|
||||
return response.status, response.read().decode()[:200]
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read().decode()[:200]
|
||||
except (ssl.SSLError, urllib.error.URLError) as error:
|
||||
return None, type(error).__name__ + ": " + str(getattr(error, "reason", error))
|
||||
|
||||
|
||||
def attempt(label: str, cert: str | None, expected, key: str | None = None):
|
||||
ctx = client_context(cert, key)
|
||||
status, body = get(GATEWAY_URL, ctx)
|
||||
ok = (status in expected) if isinstance(expected, (list, tuple, set)) else (status == expected)
|
||||
print(f"{'PASS' if ok else 'FAIL'} {label}: status={status} detail={body[:80]}")
|
||||
return ok
|
||||
|
||||
|
||||
def openbao_checks() -> bool:
|
||||
role_id = (ROOT / "secrets" / "openbao_role_id.txt").read_text().strip()
|
||||
secret_id = (ROOT / "secrets" / "openbao_secret_id.txt").read_text().strip()
|
||||
ctx = client_context(None)
|
||||
login = urllib.request.Request(
|
||||
OPENBAO + "/v1/auth/approle/login",
|
||||
data=json.dumps({"role_id": role_id, "secret_id": secret_id}).encode(),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
with urllib.request.urlopen(login, context=ctx, timeout=10) as response:
|
||||
token = json.loads(response.read())["auth"]["client_token"]
|
||||
own_status, _ = get(OPENBAO + "/v1/kv/data/messenger", ctx, token)
|
||||
other_status, _ = get(OPENBAO + "/v1/kv/data/other", ctx, token)
|
||||
ok = own_status == 200 and other_status == 403
|
||||
print(f"{'PASS' if ok else 'FAIL'} openbao policy: own={own_status} forbidden={other_status}")
|
||||
return ok
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not (PKI / "ca.crt").exists():
|
||||
raise SystemExit("PKI is missing: run scripts/pki.sh first")
|
||||
results = [
|
||||
attempt("no client certificate", None, [400, 495, 496, 497, None]),
|
||||
attempt("foreign CA certificate", "foreign-client.crt", [400, 403, 495, 496, 497, None]),
|
||||
attempt("trusted CA, wrong identity", "other.crt", [403]),
|
||||
attempt("trusted CA, LB identity", "lb.crt", [200]),
|
||||
]
|
||||
results.append(openbao_checks())
|
||||
if not all(results):
|
||||
raise SystemExit("security checks failed")
|
||||
print("PASS: trust matrix negatives and positives verified")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check the shared login rate limit (429 + Retry-After) over HTTPS."""
|
||||
import base64
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CA = str(ROOT / "pki" / "ca.crt")
|
||||
BASE = "https://localhost:8443"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
ctx = ssl.create_default_context(cafile=CA)
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method,
|
||||
headers={"Content-Type": "application/json", **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=ctx, timeout=10) as response:
|
||||
return response.status, dict(response.headers), response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, dict(error.headers), error.read()
|
||||
|
||||
|
||||
def main():
|
||||
username = "rl_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, _ = call("POST", "/api/v1/users",
|
||||
{"username": username, "display_name": "Rate limit", "password": password})
|
||||
assert status == 201, status
|
||||
bad = base64.b64encode((username + ":wrong-password-123").encode()).decode()
|
||||
results = []
|
||||
for _ in range(6):
|
||||
status, headers, body = call("POST", "/api/v1/auth/sessions",
|
||||
headers={"Authorization": "Basic " + bad})
|
||||
retry_after = headers.get("Retry-After") or headers.get("retry-after")
|
||||
results.append((status, retry_after))
|
||||
print("attempts:", results)
|
||||
limited = results[-1][0] == 429 and results[-1][1] is not None
|
||||
earlier = all(code == 401 for code, _ in results[:5])
|
||||
if not (limited and earlier):
|
||||
raise SystemExit("login rate limit did not behave as expected")
|
||||
print("PASS: login rate limit returns 429 with Retry-After after repeated failures")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable → Regular
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) SVC=api1 ;;
|
||||
api-2) SVC=api2 ;;
|
||||
*) echo "unknown instance alias: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop "$SVC"
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from . import config
|
||||
from .db import get_db
|
||||
from .models import User, Session as DbSession
|
||||
from .errors import ApiError
|
||||
|
||||
_ph = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, hash_len=32, salt_len=16)
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
def hash_password(p: str) -> str:
|
||||
return _ph.hash(p)
|
||||
|
||||
def verify_password(h: str, p: str) -> bool:
|
||||
try:
|
||||
return _ph.verify(h, p)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
def new_session(db: Session, user: User) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(48)
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
db.add(DbSession(token_hash=token_hash, user_id=user.id, expires_at=expires))
|
||||
db.commit()
|
||||
return token, expires
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("bearer "):
|
||||
raise ApiError(401, "unauthorized", "missing bearer token", challenge=_BEARER)
|
||||
token = auth[7:].strip()
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, token_hash)
|
||||
if not s:
|
||||
raise ApiError(401, "unauthorized", "invalid session", challenge=_BEARER)
|
||||
if s.expires_at <= datetime.now(timezone.utc):
|
||||
raise ApiError(401, "unauthorized", "expired session", challenge=_BEARER)
|
||||
u = db.get(User, s.user_id)
|
||||
if not u:
|
||||
raise ApiError(401, "unauthorized", "user gone", challenge=_BEARER)
|
||||
return u
|
||||
@@ -0,0 +1,60 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
def _read_secret(env_name: str, file_env: str, default=None):
|
||||
path = os.environ.get(file_env)
|
||||
if path:
|
||||
return Path(path).read_text().strip()
|
||||
return os.environ.get(env_name, default)
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
# --- Security (lab7) ---
|
||||
# When OPENBAO_ADDR is set, runtime credentials come from OpenBao via AppRole.
|
||||
# The file-based path is kept only for local unit runs without the full stand.
|
||||
OPENBAO_ADDR = os.environ.get("OPENBAO_ADDR")
|
||||
OPENBAO_CA = os.environ.get("OPENBAO_CA", "/pki/ca.crt")
|
||||
PKI_DIR = os.environ.get("PKI_DIR", "/pki")
|
||||
|
||||
if OPENBAO_ADDR:
|
||||
from .secrets import fetch_secrets
|
||||
_kv, _bao_token, _bao_lease = fetch_secrets()
|
||||
DATABASE_URL = _kv["database_url"]
|
||||
S3_ACCESS_KEY = _kv["s3_access_key_id"]
|
||||
S3_SECRET_KEY = _kv["s3_secret_access_key"]
|
||||
BROKER_URL = _kv["broker_url"]
|
||||
else:
|
||||
DATABASE_URL = _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
S3_ACCESS_KEY = _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
S3_SECRET_KEY = _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
BROKER_URL = _read_secret("BROKER_URL", "BROKER_URL_FILE", "amqp://messenger:messenger@broker:5672/")
|
||||
_bao_token = None
|
||||
_bao_lease = 0
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "https://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_CA = os.environ.get("S3_CA", f"{PKI_DIR}/ca.crt")
|
||||
|
||||
BROKER_CA = os.environ.get("BROKER_CA", f"{PKI_DIR}/ca.crt")
|
||||
|
||||
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
|
||||
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "api-unknown")
|
||||
|
||||
# --- lab5: durable broker + worker ---
|
||||
QUEUE_NAME = os.environ.get("JOB_QUEUE", "attachment_jobs")
|
||||
STAGING_PREFIX = os.environ.get("STAGING_PREFIX", "staging")
|
||||
THUMBNAIL_SIZE = int(os.environ.get("THUMBNAIL_SIZE", "256"))
|
||||
THUMBNAIL_QUALITY = int(os.environ.get("THUMBNAIL_QUALITY", "85"))
|
||||
MAX_JOB_ATTEMPTS = int(os.environ.get("MAX_JOB_ATTEMPTS", "5"))
|
||||
OUTBOX_INTERVAL_SECONDS = float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "2"))
|
||||
OUTBOX_GRACE_SECONDS = float(os.environ.get("OUTBOX_GRACE_SECONDS", "0"))
|
||||
|
||||
# --- lab7: login rate limit ---
|
||||
LOGIN_RATE_LIMIT = int(os.environ.get("LOGIN_RATE_LIMIT", "5"))
|
||||
LOGIN_RATE_WINDOW_SECONDS = int(os.environ.get("LOGIN_RATE_WINDOW_SECONDS", "60"))
|
||||
@@ -0,0 +1,16 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, DeclarativeBase
|
||||
from . import config
|
||||
|
||||
engine = create_engine(config.DATABASE_URL, pool_pre_ping=True, pool_size=5, max_overflow=10)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,33 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, challenge: str | None = None,
|
||||
retry_after: int | None = None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.challenge = challenge
|
||||
self.retry_after = retry_after
|
||||
|
||||
def _rid(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", None) or "unknown"
|
||||
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
headers = {"X-Request-Id": _rid(request)}
|
||||
if exc.challenge:
|
||||
headers["WWW-Authenticate"] = exc.challenge
|
||||
if exc.retry_after is not None:
|
||||
headers["Retry-After"] = str(exc.retry_after)
|
||||
return JSONResponse(
|
||||
status_code=exc.status,
|
||||
content={"error": {"code": exc.code, "message": exc.message, "request_id": _rid(request)}},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
async def unhandled_handler(request: Request, exc: Exception):
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": {"code": "internal_error", "message": "internal error", "request_id": _rid(request)}},
|
||||
headers={"X-Request-Id": _rid(request)},
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import io
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from . import config
|
||||
from .errors import ApiError
|
||||
|
||||
def validate_image(data: bytes) -> tuple[str, int, int, int]:
|
||||
"""Возвращает (content_type, width, height, size)."""
|
||||
if not data:
|
||||
raise ApiError(400, "invalid_request", "empty body")
|
||||
if len(data) > config.MAX_UPLOAD_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "file too large")
|
||||
head = data[:8]
|
||||
if head.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
ctype = "image/png"
|
||||
elif head.startswith(b"\xff\xd8\xff"):
|
||||
ctype = "image/jpeg"
|
||||
else:
|
||||
raise ApiError(415, "unsupported_media_type", "only PNG/JPEG")
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
w, h = img.size
|
||||
except UnidentifiedImageError:
|
||||
raise ApiError(415, "unsupported_media_type", "invalid image")
|
||||
if w * h > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "too many pixels")
|
||||
return ctype, w, h, len(data)
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
"""Broker access: durable job publication and the transactional-outbox pump.
|
||||
|
||||
The broker message carries only identifiers and metadata (never bytes or
|
||||
credentials). The job row in PostgreSQL is the source of truth; `published_at`
|
||||
turns the row into a small transactional outbox so a crash between COMMIT and
|
||||
publish is repaired by the background pump.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib.parse import urlparse, unquote
|
||||
|
||||
import pika
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config
|
||||
from .db import SessionLocal
|
||||
from .models import AttachmentJob
|
||||
|
||||
log = logging.getLogger("jobs")
|
||||
|
||||
TYPE_V1 = "attachment.process.v1"
|
||||
|
||||
|
||||
def pika_params() -> pika.ConnectionParameters:
|
||||
"""TLS parameters for the broker (amqps) with server verification."""
|
||||
parsed = urlparse(config.BROKER_URL)
|
||||
if parsed.scheme == "amqps":
|
||||
context = ssl.create_default_context(cafile=config.BROKER_CA)
|
||||
context.check_hostname = True
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
ssl_options = pika.SSLOptions(context, server_hostname=parsed.hostname)
|
||||
else:
|
||||
ssl_options = None
|
||||
params = pika.ConnectionParameters(
|
||||
host=parsed.hostname or "broker",
|
||||
port=parsed.port or (5671 if parsed.scheme == "amqps" else 5672),
|
||||
credentials=pika.PlainCredentials(unquote(parsed.username or "guest"),
|
||||
unquote(parsed.password or "guest")),
|
||||
ssl_options=ssl_options,
|
||||
heartbeat=30,
|
||||
blocked_connection_timeout=30,
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def build_payload(job: AttachmentJob, attempt: int, carrier: dict | None = None) -> dict:
|
||||
from opentelemetry.propagate import inject
|
||||
if carrier is None:
|
||||
carrier = {}
|
||||
inject(carrier) # W3C traceparent when a request span is active
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job.id),
|
||||
"type": TYPE_V1,
|
||||
"attachment_id": str(job.attachment_id),
|
||||
"pipeline_version": job.pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
if carrier.get("traceparent"):
|
||||
payload["traceparent"] = carrier["traceparent"]
|
||||
if carrier.get("tracestate"):
|
||||
payload["tracestate"] = carrier["tracestate"]
|
||||
return payload
|
||||
|
||||
|
||||
def publish(payload: dict) -> None:
|
||||
connection = pika.BlockingConnection(pika_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.confirm_delivery()
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=config.QUEUE_NAME,
|
||||
body=json.dumps(payload).encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2, # persistent
|
||||
content_type="application/json",
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def publish_job_row(job_id: uuid.UUID, attempt: int, carrier: dict | None = None) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job or job.state in ("done", "failed"):
|
||||
return
|
||||
payload = build_payload(job, attempt, carrier)
|
||||
publish(payload)
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if job:
|
||||
job.published_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def pump_once(limit: int = 50) -> int:
|
||||
"""Publish queued rows whose outbox timestamp is still empty."""
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=config.OUTBOX_GRACE_SECONDS)
|
||||
published = 0
|
||||
with SessionLocal() as db:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(AttachmentJob)
|
||||
.where(AttachmentJob.published_at.is_(None))
|
||||
.where(AttachmentJob.state == "queued")
|
||||
.where(AttachmentJob.created_at <= cutoff)
|
||||
.order_by(AttachmentJob.created_at)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
pending = [(job.id, job.attempt + 1) for job in rows]
|
||||
for job_id, attempt in pending:
|
||||
try:
|
||||
publish_job_row(job_id, attempt)
|
||||
published += 1
|
||||
except Exception as exc: # broker may still be starting
|
||||
log.warning("outbox publish failed for %s: %s", job_id, exc)
|
||||
return published
|
||||
|
||||
|
||||
def start_outbox_thread() -> threading.Thread:
|
||||
def loop():
|
||||
while True:
|
||||
try:
|
||||
count = pump_once()
|
||||
if count:
|
||||
log.info("outbox published %d job(s)", count)
|
||||
except Exception as exc:
|
||||
log.warning("outbox pump error: %s", exc)
|
||||
time.sleep(config.OUTBOX_INTERVAL_SECONDS)
|
||||
|
||||
thread = threading.Thread(target=loop, name="outbox", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
from . import config, jobs, telemetry
|
||||
from .db import engine, Base
|
||||
from .errors import ApiError, api_error_handler, unhandled_handler
|
||||
from .s3 import ensure_bucket
|
||||
from .routers import health, users, sessions, chats, messages, attachments
|
||||
|
||||
tracer_provider, meter_provider = telemetry.setup_telemetry("messenger-api", config.INSTANCE_ID)
|
||||
request_counter, request_duration = telemetry.http_instruments()
|
||||
logger = logging.getLogger("messenger.api")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# create_all is not idempotent under parallel startup: two replicas
|
||||
# may race and one hits UniqueViolation on pg_type. Catch and re-run
|
||||
# once after a short delay; by then the other replica has finished.
|
||||
import asyncio
|
||||
from sqlalchemy.exc import IntegrityError, OperationalError
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
break
|
||||
except (IntegrityError, OperationalError) as exc:
|
||||
print(f"WARN: create_all attempt {attempt+1} failed: {exc}")
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
print("WARN: create_all failed after 5 attempts; continuing")
|
||||
|
||||
try:
|
||||
ensure_bucket()
|
||||
except Exception as exc:
|
||||
print(f"WARN: ensure_bucket failed: {exc}")
|
||||
jobs.start_outbox_thread()
|
||||
logger.info("api.started", extra={"event_fields": json.dumps({"instance_id": config.INSTANCE_ID})})
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.add_exception_handler(ApiError, api_error_handler)
|
||||
app.add_exception_handler(Exception, unhandled_handler)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def observe_request(request: Request, call_next):
|
||||
started = time.perf_counter()
|
||||
response = await call_next(request)
|
||||
route = request.scope.get("route")
|
||||
template = getattr(route, "path", request.url.path)
|
||||
attributes = {
|
||||
"http.request.method": request.method,
|
||||
"http.route": template,
|
||||
"http.response.status_code": response.status_code,
|
||||
}
|
||||
request_counter.add(1, attributes)
|
||||
request_duration.record(time.perf_counter() - started, attributes)
|
||||
logger.info("http.request", extra={"event_fields": json.dumps({
|
||||
"http.request.method": request.method,
|
||||
"http.route": template,
|
||||
"http.response.status_code": response.status_code,
|
||||
"request_id": getattr(request.state, "request_id", None)
|
||||
or request.headers.get("X-Request-Id"),
|
||||
})})
|
||||
return response
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex
|
||||
request.state.request_id = rid
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
for r in (health, users, sessions, chats, messages, attachments):
|
||||
app.include_router(r.router)
|
||||
|
||||
FastAPIInstrumentor.instrument_app(app, excluded_urls="health/live,health/ready")
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import String, DateTime, ForeignKey, Integer, UniqueConstraint, Index
|
||||
from sqlalchemy.dialects.postgresql import UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .db import Base
|
||||
|
||||
|
||||
def _uuid() -> uuid.UUID:
|
||||
return uuid.uuid4()
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100))
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
|
||||
class Session(Base):
|
||||
__tablename__ = "sessions"
|
||||
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True) # sha256(token) hex
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
title: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
text: Mapped[str] = mapped_column(String(4000), default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
__tablename__ = "idempotency_keys"
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
method: Mapped[str] = mapped_column(String(8), primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
request_hash: Mapped[str] = mapped_column(String(64))
|
||||
response_json: Mapped[str] = mapped_column(String)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
__table_args__ = (Index("ix_idem_created", "created_at"),)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="ready")
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
"""Durable intent to process one attachment (transactional outbox row)."""
|
||||
__tablename__ = "attachment_jobs"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
state: Mapped[str] = mapped_column(String(16), default="queued") # queued|processing|done|failed
|
||||
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentVariant(Base):
|
||||
__tablename__ = "attachment_variants"
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
width: Mapped[int] = mapped_column(Integer)
|
||||
height: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
|
||||
class LoginAttempt(Base):
|
||||
__tablename__ = "login_attempts"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
key: Mapped[str] = mapped_column(String(128), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
@@ -0,0 +1,122 @@
|
||||
import hashlib
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends, Request, Query
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import Attachment, ChatMember, AttachmentJob, AttachmentVariant
|
||||
from ..auth import current_user
|
||||
from ..errors import ApiError
|
||||
from .. import s3, config, jobs
|
||||
from ..images import validate_image
|
||||
from ..schemas import AttachmentOut, VariantOut
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
def _bad_filename(name: str) -> bool:
|
||||
return (not name) or len(name) > 255 or any(c in name for c in ("/", "\\", "\x00")) or any(ord(c) < 32 for c in name)
|
||||
|
||||
def _member_chat_or_404(db, chat_id, user_id):
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
def _attachment_for_member(db, att_id, user_id) -> Attachment:
|
||||
att = db.get(Attachment, att_id)
|
||||
if not att:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
_member_chat_or_404(db, att.chat_id, user_id)
|
||||
return att
|
||||
|
||||
def _attachment_out(db, att: Attachment) -> AttachmentOut:
|
||||
variants = db.query(AttachmentVariant).filter(
|
||||
AttachmentVariant.attachment_id == att.id
|
||||
).all()
|
||||
return AttachmentOut(
|
||||
id=att.id, chat_id=att.chat_id, owner_id=att.owner_id,
|
||||
filename=att.filename, content_type=att.content_type,
|
||||
size_bytes=att.size_bytes, sha256=att.sha256,
|
||||
status=att.status, error_code=att.error_code,
|
||||
variants=[VariantOut.model_validate(v) for v in variants],
|
||||
created_at=att.created_at, updated_at=att.updated_at,
|
||||
)
|
||||
|
||||
@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut)
|
||||
async def upload(chat_id: uuid.UUID,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
request: Request = None,
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(current_user)):
|
||||
_member_chat_or_404(db, chat_id, user.id)
|
||||
if _bad_filename(filename):
|
||||
raise ApiError(400, "invalid_request", "invalid filename")
|
||||
ctype_header = request.headers.get("content-type", "").split(";")[0].strip()
|
||||
if ctype_header not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "expected image/png or image/jpeg")
|
||||
data = await request.body()
|
||||
ctype, w, h, size = validate_image(data)
|
||||
if ctype != ctype_header:
|
||||
raise ApiError(415, "unsupported_media_type", "content-type mismatch with file")
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
|
||||
att_id = uuid.uuid4()
|
||||
final_key = f"chats/{chat_id}/attachments/{att_id}"
|
||||
staging_key = f"{config.STAGING_PREFIX}/{att_id}"
|
||||
|
||||
# 1. Durable staging: the original must survive an API restart before 202.
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=staging_key, Body=data, ContentType=ctype)
|
||||
|
||||
# 2. Metadata + durable job intent in one transaction (transactional outbox).
|
||||
att = Attachment(id=att_id, chat_id=chat_id, owner_id=user.id, filename=filename,
|
||||
content_type=ctype, size_bytes=size, sha256=digest, s3_key=final_key,
|
||||
staging_key=staging_key, status="queued")
|
||||
job = AttachmentJob(attachment_id=att_id, pipeline_version=1, state="queued", attempt=0)
|
||||
try:
|
||||
db.add(att)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(att)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# Compensating delete for the staging object; the DB has no intent.
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
# 3. Handoff to the broker. If this fails, the row stays unpublished and
|
||||
# the outbox pump retries it; the client still got a durable 202.
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from opentelemetry.propagate import inject
|
||||
carrier: dict[str, str] = {}
|
||||
inject(carrier) # capture the active request span before leaving the event loop
|
||||
try:
|
||||
await run_in_threadpool(jobs.publish_job_row, job.id, 1, carrier)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _attachment_out(db, att)
|
||||
|
||||
@router.get("/attachments/{att_id}", response_model=AttachmentOut)
|
||||
def meta(att_id: uuid.UUID, db=Depends(get_db), user=Depends(current_user)):
|
||||
return _attachment_out(db, _attachment_for_member(db, att_id, user.id))
|
||||
|
||||
@router.get("/attachments/{att_id}/content")
|
||||
def content(att_id: uuid.UUID, variant: str | None = None,
|
||||
db=Depends(get_db), user=Depends(current_user)):
|
||||
att = _attachment_for_member(db, att_id, user.id)
|
||||
if att.status == "ready":
|
||||
if variant in (None, "original"):
|
||||
key, media_type = att.s3_key, att.content_type
|
||||
else:
|
||||
row = db.get(AttachmentVariant, (att.id, variant))
|
||||
if not row:
|
||||
raise ApiError(404, "not_found", "variant not found")
|
||||
key, media_type = row.s3_key, row.content_type
|
||||
elif att.status in ("queued", "processing"):
|
||||
raise ApiError(409, "conflict", "attachment is not ready")
|
||||
else:
|
||||
raise ApiError(409, "conflict", "attachment failed")
|
||||
obj = s3.client().get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
body = obj["Body"].read()
|
||||
return Response(content=body, media_type=media_type)
|
||||
@@ -0,0 +1,44 @@
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import User, Chat, ChatMember
|
||||
from ..schemas import CreateChatIn, ChatOut, ChatPage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
def _chat_out(db, chat) -> ChatOut:
|
||||
members = [m.user_id for m in db.query(ChatMember).filter(ChatMember.chat_id == chat.id).all()]
|
||||
return ChatOut(id=chat.id, title=chat.title, member_ids=members, created_at=chat.created_at)
|
||||
|
||||
@router.post("", status_code=201, response_model=ChatOut)
|
||||
def create_chat(body: CreateChatIn, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if user.id not in body.member_ids:
|
||||
raise ApiError(400, "invalid_request", "caller must be a member")
|
||||
found = db.query(User.id).filter(User.id.in_(body.member_ids)).all()
|
||||
if len(found) != len(set(body.member_ids)):
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
chat = Chat(title=body.title)
|
||||
db.add(chat); db.flush()
|
||||
for uid in body.member_ids:
|
||||
db.add(ChatMember(chat_id=chat.id, user_id=uid))
|
||||
db.commit(); db.refresh(chat)
|
||||
return _chat_out(db, chat)
|
||||
|
||||
@router.get("", response_model=ChatPage)
|
||||
def list_chats(db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
rows = db.query(Chat).join(ChatMember, ChatMember.chat_id == Chat.id)\
|
||||
.filter(ChatMember.user_id == user.id)\
|
||||
.order_by(Chat.created_at, Chat.id).limit(100).all()
|
||||
return ChatPage(items=[_chat_out(db, c) for c in rows], next_cursor=None)
|
||||
|
||||
@router.get("/{chat_id}", response_model=ChatOut)
|
||||
def get_chat(chat_id: uuid.UUID, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if not db.get(ChatMember, (chat_id, user.id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
chat = db.get(Chat, chat_id)
|
||||
if not chat:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return _chat_out(db, chat)
|
||||
@@ -0,0 +1,19 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..errors import ApiError
|
||||
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
@router.get("/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@router.get("/ready")
|
||||
def ready(db: Session = Depends(get_db)):
|
||||
try:
|
||||
db.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
raise ApiError(503, "unavailable", "database unavailable")
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,147 @@
|
||||
import uuid
|
||||
import json
|
||||
import hashlib
|
||||
import base64
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..db import get_db
|
||||
from ..models import (
|
||||
User,
|
||||
ChatMember,
|
||||
Message,
|
||||
MessageAttachment,
|
||||
IdempotencyKey,
|
||||
Attachment,
|
||||
)
|
||||
from ..schemas import CreateMessageIn, MessageOut, MessagePage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
|
||||
|
||||
|
||||
def _msg_out(db: Session, m: Message) -> MessageOut:
|
||||
atts = [
|
||||
a.attachment_id
|
||||
for a in db.query(MessageAttachment)
|
||||
.filter(MessageAttachment.message_id == m.id)
|
||||
.order_by(MessageAttachment.position)
|
||||
.all()
|
||||
]
|
||||
return MessageOut(
|
||||
id=m.id,
|
||||
chat_id=m.chat_id,
|
||||
sender_id=m.sender_id,
|
||||
text=m.text,
|
||||
created_at=m.created_at,
|
||||
attachment_ids=atts,
|
||||
)
|
||||
|
||||
|
||||
def _require_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=MessageOut)
|
||||
def send(
|
||||
chat_id: uuid.UUID,
|
||||
body: CreateMessageIn,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
key = request.headers.get("idempotency-key", "")
|
||||
if not key or len(key) > 128:
|
||||
raise ApiError(400, "invalid_request", "Idempotency-Key required")
|
||||
|
||||
if not body.text.strip() and not body.attachment_ids:
|
||||
raise ApiError(400, "invalid_request", "empty message")
|
||||
|
||||
for aid in body.attachment_ids:
|
||||
a = db.get(Attachment, aid)
|
||||
if not a or a.chat_id != chat_id:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(body.model_dump(), sort_keys=True, default=str).encode()
|
||||
).hexdigest()
|
||||
|
||||
existing = db.get(IdempotencyKey, (user.id, "POST", path, key))
|
||||
if existing:
|
||||
if existing.request_hash != body_hash:
|
||||
raise ApiError(409, "conflict", "idempotency key reused with different body")
|
||||
return json.loads(existing.response_json)
|
||||
|
||||
m = Message(chat_id=chat_id, sender_id=user.id, text=body.text)
|
||||
db.add(m)
|
||||
db.flush() # чтобы появился m.id
|
||||
|
||||
for i, aid in enumerate(body.attachment_ids):
|
||||
db.add(MessageAttachment(message_id=m.id, attachment_id=aid, position=i))
|
||||
db.flush() # чтобы _msg_out увидел MessageAttachment
|
||||
|
||||
out = _msg_out(db, m)
|
||||
|
||||
db.add(
|
||||
IdempotencyKey(
|
||||
user_id=user.id,
|
||||
method="POST",
|
||||
path=path,
|
||||
key=key,
|
||||
request_hash=body_hash,
|
||||
response_json=json.dumps(out.model_dump(), default=str),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return out
|
||||
|
||||
|
||||
@router.get("", response_model=MessagePage)
|
||||
def list_messages(
|
||||
chat_id: uuid.UUID,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
raise ApiError(400, "invalid_request", "bad limit")
|
||||
|
||||
q = (
|
||||
db.query(Message)
|
||||
.filter(Message.chat_id == chat_id)
|
||||
.order_by(Message.created_at, Message.id)
|
||||
)
|
||||
|
||||
if cursor:
|
||||
try:
|
||||
c = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
|
||||
ts = datetime.fromisoformat(c["t"])
|
||||
mid = uuid.UUID(c["i"])
|
||||
q = q.filter(
|
||||
(Message.created_at > ts)
|
||||
| ((Message.created_at == ts) & (Message.id > mid))
|
||||
)
|
||||
except Exception:
|
||||
raise ApiError(400, "invalid_request", "bad cursor")
|
||||
|
||||
rows = q.limit(limit + 1).all()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = rows[-1]
|
||||
payload = json.dumps({"t": last.created_at.isoformat(), "i": str(last.id)}).encode()
|
||||
next_cursor = base64.urlsafe_b64encode(payload).decode()
|
||||
|
||||
return MessagePage(items=[_msg_out(db, m) for m in rows], next_cursor=next_cursor)
|
||||
@@ -0,0 +1,72 @@
|
||||
import base64
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from .. import config
|
||||
from ..db import get_db
|
||||
from ..models import User, Session as DbSession, LoginAttempt
|
||||
from ..schemas import SessionOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import verify_password, new_session, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
|
||||
|
||||
_BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
forwarded = request.headers.get("x-forwarded-for", "")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()[:64]
|
||||
return (request.client.host if request.client else "unknown")[:64]
|
||||
|
||||
|
||||
def _enforce_login_rate(db: Session, key: str) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
window_start = now - timedelta(seconds=config.LOGIN_RATE_WINDOW_SECONDS)
|
||||
attempts = (
|
||||
db.query(LoginAttempt)
|
||||
.filter(LoginAttempt.key == key)
|
||||
.filter(LoginAttempt.created_at > window_start)
|
||||
.count()
|
||||
)
|
||||
if attempts >= config.LOGIN_RATE_LIMIT:
|
||||
raise ApiError(429, "rate_limited", "too many login attempts", retry_after=config.LOGIN_RATE_WINDOW_SECONDS)
|
||||
|
||||
|
||||
def _record_failure(db: Session, key: str) -> None:
|
||||
db.add(LoginAttempt(key=key))
|
||||
db.commit()
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=SessionOut)
|
||||
def login(request: Request, db: Session = Depends(get_db)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("basic "):
|
||||
raise ApiError(401, "unauthorized", "basic required", challenge=_BASIC)
|
||||
try:
|
||||
raw = base64.b64decode(auth[6:].strip()).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
except Exception:
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
key = f"{username}:{_client_ip(request)}"
|
||||
_enforce_login_rate(db, key)
|
||||
user = db.query(User).filter(User.username == username).first()
|
||||
if not user or not verify_password(user.password_hash, password):
|
||||
_record_failure(db, key)
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
token, expires = new_session(db, user)
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": expires, "user": user}
|
||||
|
||||
@router.delete("/current", status_code=204)
|
||||
def logout(request: Request, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
token = auth[7:].strip()
|
||||
h = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, h)
|
||||
if s:
|
||||
db.delete(s)
|
||||
db.commit()
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from ..db import get_db
|
||||
from ..models import User
|
||||
from ..schemas import CreateUserIn, UserOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import hash_password, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
@router.post("", status_code=201, response_model=UserOut)
|
||||
def register(body: CreateUserIn, db: Session = Depends(get_db)):
|
||||
u = User(username=body.username, display_name=body.display_name,
|
||||
password_hash=hash_password(body.password))
|
||||
db.add(u)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise ApiError(409, "conflict", "username taken")
|
||||
db.refresh(u)
|
||||
return u
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(user: User = Depends(current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,37 @@
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from . import config
|
||||
|
||||
_client = None
|
||||
|
||||
def client():
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
verify=config.S3_CA,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
connect_timeout=3,
|
||||
read_timeout=10,
|
||||
),
|
||||
)
|
||||
return _client
|
||||
|
||||
def ensure_bucket():
|
||||
c = client()
|
||||
try:
|
||||
c.head_bucket(Bucket=config.S3_BUCKET)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
c.create_bucket(Bucket=config.S3_BUCKET)
|
||||
else:
|
||||
raise
|
||||
@@ -0,0 +1,76 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
class CreateUserIn(BaseModel):
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
class SessionOut(BaseModel):
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: UserOut
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
title: str
|
||||
member_ids: list[uuid.UUID]
|
||||
created_at: datetime
|
||||
|
||||
class CreateChatIn(BaseModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
sender_id: uuid.UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[uuid.UUID]
|
||||
|
||||
class CreateMessageIn(BaseModel):
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[MessageOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[ChatOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class VariantOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
class AttachmentOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
owner_id: uuid.UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list[VariantOut] = []
|
||||
error_code: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Service-identity secret loading from OpenBao (AppRole -> KV v2).
|
||||
|
||||
The application never uses a root token or unseal keys. It logs in with a
|
||||
short-lived SecretID issued by the operator bootstrap, receives a TTL-bound
|
||||
token and reads only its own KV path.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read(path: str) -> str:
|
||||
return Path(path).read_text().strip()
|
||||
|
||||
|
||||
def _context(ca_file: str) -> ssl.SSLContext:
|
||||
ctx = ssl.create_default_context(cafile=ca_file)
|
||||
ctx.check_hostname = True
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
return ctx
|
||||
|
||||
|
||||
def _call(method: str, url: str, ctx: ssl.SSLContext, token: str | None = None,
|
||||
body: dict | None = None, timeout: int = 10):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
request = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
with urllib.request.urlopen(request, context=ctx, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode())
|
||||
|
||||
|
||||
def fetch_secrets() -> tuple[dict, str, int]:
|
||||
"""Return (kv data, token, lease_seconds) for the messenger path."""
|
||||
addr = os.environ.get("OPENBAO_ADDR", "https://openbao:8200")
|
||||
ca = os.environ.get("OPENBAO_CA", "/pki/ca.crt")
|
||||
role_id_file = os.environ.get("OPENBAO_ROLE_ID_FILE", "/run/secrets/openbao_role_id")
|
||||
secret_id_file = os.environ.get("OPENBAO_SECRET_ID_FILE", "/run/secrets/openbao_secret_id")
|
||||
|
||||
ctx = _context(ca)
|
||||
login = _call("POST", addr + "/v1/auth/approle/login", ctx, body={
|
||||
"role_id": _read(role_id_file),
|
||||
"secret_id": _read(secret_id_file),
|
||||
})
|
||||
token = login["auth"]["client_token"]
|
||||
lease = int(login["auth"].get("lease_duration", 3600))
|
||||
kv = _call("GET", addr + "/v1/kv/data/messenger", ctx, token=token)
|
||||
return kv["data"]["data"], token, lease
|
||||
@@ -0,0 +1,136 @@
|
||||
"""OpenTelemetry wiring for API and worker (lab6).
|
||||
|
||||
Signals go OTLP -> collector. Metrics are exposed by the collector as a
|
||||
Prometheus endpoint, traces are stored in Tempo, logs in Loki. The JSON logs
|
||||
kept in stdout also carry trace_id/span_id so a trace can be matched with a
|
||||
line even without the log backend.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import (
|
||||
Resource,
|
||||
SERVICE_INSTANCE_ID,
|
||||
SERVICE_NAME,
|
||||
SERVICE_VERSION,
|
||||
)
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.propagate import set_global_textmap
|
||||
|
||||
|
||||
class TraceJsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line, with trace correlation and no bodies."""
|
||||
|
||||
def __init__(self, service_name: str):
|
||||
super().__init__()
|
||||
self.service_name = service_name
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
payload = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"severity": record.levelname,
|
||||
"service.name": self.service_name,
|
||||
"event": record.getMessage(),
|
||||
"logger": record.name,
|
||||
}
|
||||
if ctx and ctx.is_valid:
|
||||
payload["trace_id"] = format(ctx.trace_id, "032x")
|
||||
payload["span_id"] = format(ctx.span_id, "016x")
|
||||
extra = getattr(record, "event_fields", None)
|
||||
if isinstance(extra, dict):
|
||||
payload.update({k: v for k, v in extra.items() if not isinstance(v, (bytes, bytearray))})
|
||||
elif isinstance(extra, str):
|
||||
try:
|
||||
payload.update({k: v for k, v in json.loads(extra).items()})
|
||||
except ValueError:
|
||||
pass
|
||||
if record.exc_info and record.exc_info[0]:
|
||||
payload["error.type"] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def _otlp_credentials():
|
||||
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
|
||||
if not endpoint.startswith("https://"):
|
||||
return endpoint, None
|
||||
import grpc
|
||||
ca_path = os.environ.get("OTEL_CA", "/pki/ca.crt")
|
||||
with open(ca_path, "rb") as handle:
|
||||
root = handle.read()
|
||||
return endpoint, grpc.ssl_channel_credentials(root_certificates=root)
|
||||
|
||||
|
||||
def setup_telemetry(service_name: str, instance_id: str):
|
||||
endpoint, credentials = _otlp_credentials()
|
||||
insecure = credentials is None
|
||||
resource = Resource.create({
|
||||
SERVICE_NAME: service_name,
|
||||
SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "6.0.0"),
|
||||
SERVICE_INSTANCE_ID: instance_id,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
})
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, credentials=credentials, insecure=insecure))
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint, credentials=credentials, insecure=insecure),
|
||||
export_interval_millis=int(os.environ.get("OTEL_METRIC_INTERVAL_MS", "10000")),
|
||||
)],
|
||||
)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint, credentials=credentials, insecure=insecure))
|
||||
)
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.INFO)
|
||||
for handler in list(root.handlers):
|
||||
root.removeHandler(handler)
|
||||
stream = logging.StreamHandler()
|
||||
stream.setFormatter(TraceJsonFormatter(service_name))
|
||||
root.addHandler(stream)
|
||||
root.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logger_provider))
|
||||
|
||||
set_global_textmap(TraceContextTextMapPropagator())
|
||||
return tracer_provider, meter_provider
|
||||
|
||||
|
||||
def http_instruments():
|
||||
meter = metrics.get_meter("messenger.http")
|
||||
return (
|
||||
meter.create_counter("http.server.requests", unit="1",
|
||||
description="HTTP requests by method, route template and status"),
|
||||
meter.create_histogram("http.server.request.duration", unit="s",
|
||||
description="HTTP server request duration"),
|
||||
)
|
||||
|
||||
|
||||
def job_instruments():
|
||||
meter = metrics.get_meter("messenger.worker")
|
||||
return (
|
||||
meter.create_counter("attachment.jobs", unit="1",
|
||||
description="Attachment jobs by outcome"),
|
||||
meter.create_histogram("attachment.job.duration", unit="s",
|
||||
description="Attachment job duration by outcome"),
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Deterministic image variants (lab5: one thumbnail per the contract)."""
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def _flatten_on_white(img: Image.Image) -> Image.Image:
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
|
||||
return Image.alpha_composite(background, rgba).convert("RGB")
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def make_thumbnail(data: bytes) -> tuple[bytes, int, int, str]:
|
||||
"""Fit the image into 256x256 (no upscaling), JPEG on white.
|
||||
|
||||
Returns (jpeg_bytes, width, height, sha256_hex).
|
||||
"""
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = _flatten_on_white(img)
|
||||
|
||||
limit = config.THUMBNAIL_SIZE
|
||||
width, height = img.size
|
||||
scale = min(limit / width, limit / height, 1.0)
|
||||
new_width = max(1, int(width * scale))
|
||||
new_height = max(1, int(height * scale))
|
||||
if (new_width, new_height) != (width, height):
|
||||
img = img.resize((new_width, new_height), Image.LANCZOS)
|
||||
|
||||
out = io.BytesIO()
|
||||
img.save(out, "JPEG", quality=config.THUMBNAIL_QUALITY, optimize=True)
|
||||
payload = out.getvalue()
|
||||
return payload, new_width, new_height, hashlib.sha256(payload).hexdigest()
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
"""Durable attachment worker (lab5/lab6).
|
||||
|
||||
Consumes ``attachment.process.v1`` messages from a persistent queue, performs
|
||||
the heavy image work outside the HTTP path, records the result atomically in
|
||||
PostgreSQL, and emits OTLP traces/metrics/logs. Delivery is at-least-once; the
|
||||
effect is idempotent by ``(attachment_id, pipeline_version, variant)`` and
|
||||
every state transition is committed before the broker ack.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.propagate import extract
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config, s3, telemetry
|
||||
from .db import SessionLocal
|
||||
from .errors import ApiError
|
||||
from .images import validate_image
|
||||
from .models import Attachment, AttachmentJob, AttachmentVariant
|
||||
from .thumbnails import make_thumbnail
|
||||
|
||||
_, _meter_provider = telemetry.setup_telemetry("messenger-worker", "worker")
|
||||
job_counter, job_duration = telemetry.job_instruments()
|
||||
tracer = trace.get_tracer("messenger.worker")
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
global _stop
|
||||
_stop = True
|
||||
log.info("worker.stop_requested")
|
||||
|
||||
|
||||
def _failure_code(exc: Exception) -> tuple[str, bool]:
|
||||
"""Return (safe error_code, permanent)."""
|
||||
if isinstance(exc, ApiError):
|
||||
if exc.status in (400, 413, 415):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
if isinstance(exc, (OSError, ValueError)):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
|
||||
|
||||
def _set_state(job_id: uuid.UUID, attachment_id: uuid.UUID, job_state: str,
|
||||
att_state: str | None = None, attempt: int | None = None,
|
||||
error: str | None = None) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job:
|
||||
return
|
||||
job.state = job_state
|
||||
if attempt is not None:
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
if error is not None:
|
||||
job.last_error = error[:255]
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if att and att_state:
|
||||
att.status = att_state
|
||||
att.error_code = error if att_state == "failed" else None
|
||||
db.commit()
|
||||
|
||||
|
||||
def process(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> str:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if not job or not att:
|
||||
log.warning("job.dropped", extra={"event_fields": json.dumps({"reason": "row_gone"})})
|
||||
return "dropped"
|
||||
if att.status == "ready":
|
||||
job.state = "done"
|
||||
db.commit()
|
||||
log.info("job.duplicate", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
|
||||
return "duplicate"
|
||||
if att.status == "failed":
|
||||
job.state = "failed"
|
||||
db.commit()
|
||||
log.info("job.duplicate_failed", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
|
||||
return "skipped"
|
||||
job.state = "processing"
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
att.status = "processing"
|
||||
db.commit()
|
||||
pipeline_version = job.pipeline_version
|
||||
staging_key = att.staging_key or f"{config.STAGING_PREFIX}/{attachment_id}"
|
||||
final_key = att.s3_key
|
||||
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
declared_type = att.content_type if att else None
|
||||
data = s3.client().get_object(Bucket=config.S3_BUCKET, Key=staging_key)["Body"].read()
|
||||
ctype, _w, _h, _size = validate_image(data)
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=final_key, Body=data,
|
||||
ContentType=declared_type or ctype)
|
||||
|
||||
thumb, tw, th, tsha = make_thumbnail(data)
|
||||
thumb_key = f"variants/{attachment_id}/thumbnail"
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=thumb_key, Body=thumb, ContentType="image/jpeg")
|
||||
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
existing = db.get(AttachmentVariant, (attachment_id, "thumbnail"))
|
||||
if existing is None:
|
||||
db.add(AttachmentVariant(
|
||||
attachment_id=attachment_id, name="thumbnail",
|
||||
content_type="image/jpeg", size_bytes=len(thumb),
|
||||
width=tw, height=th, sha256=tsha, s3_key=thumb_key,
|
||||
))
|
||||
elif existing.s3_key != thumb_key:
|
||||
log.warning("variant.kept", extra={"event_fields": json.dumps({"reason": "newer_pipeline"})})
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
job.state = "done"
|
||||
job.last_error = None
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
db.commit()
|
||||
log.info("job.ready", extra={"event_fields": json.dumps({
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": attempt,
|
||||
})})
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
return "ready"
|
||||
except Exception as exc:
|
||||
code, permanent = _failure_code(exc)
|
||||
log.exception("job.failed", extra={"event_fields": json.dumps({
|
||||
"attachment_id": str(attachment_id),
|
||||
"attempt": attempt,
|
||||
"error_code": code,
|
||||
})})
|
||||
if permanent or attempt >= config.MAX_JOB_ATTEMPTS:
|
||||
_set_state(job_id, attachment_id, "failed", att_state="failed",
|
||||
attempt=attempt, error=code)
|
||||
return "failed"
|
||||
_set_state(job_id, attachment_id, "queued", att_state="queued",
|
||||
attempt=attempt, error=code)
|
||||
raise
|
||||
|
||||
|
||||
def handle(channel, method, properties, body):
|
||||
outcome = "retry"
|
||||
try:
|
||||
message = json.loads(body)
|
||||
job_id = uuid.UUID(message["job_id"])
|
||||
attachment_id = uuid.UUID(message["attachment_id"])
|
||||
attempt = int(message.get("attempt", 1))
|
||||
except Exception as exc:
|
||||
log.warning("job.poison", extra={"event_fields": json.dumps({"reason": type(exc).__name__})})
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
return
|
||||
|
||||
context = extract(message)
|
||||
started = time.perf_counter()
|
||||
with tracer.start_as_current_span("attachment.process", context=context,
|
||||
kind=SpanKind.CONSUMER) as span:
|
||||
span.set_attribute("job.type", JOB_TYPE)
|
||||
span.set_attribute("attachment.id", str(attachment_id))
|
||||
span.set_attribute("job.attempt", attempt)
|
||||
try:
|
||||
outcome = process(job_id, attachment_id, attempt)
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
if outcome == "failed":
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
except Exception as exc:
|
||||
outcome = "retry"
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
span.record_exception(exc)
|
||||
from .jobs import build_payload, publish
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
payload = build_payload(job, job.attempt + 1) if job and job.state == "queued" else None
|
||||
if payload:
|
||||
try:
|
||||
publish(payload)
|
||||
except Exception:
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
||||
outcome = "requeued"
|
||||
return
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
finally:
|
||||
attributes = {"job.type": JOB_TYPE, "job.outcome": outcome}
|
||||
job_counter.add(1, attributes)
|
||||
job_duration.record(time.perf_counter() - started, attributes)
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
from .jobs import pika_params
|
||||
while not _stop:
|
||||
try:
|
||||
connection = pika.BlockingConnection(pika_params())
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=config.QUEUE_NAME, on_message_callback=handle, auto_ack=False)
|
||||
log.info("worker.consuming", extra={"event_fields": json.dumps({"queue": config.QUEUE_NAME})})
|
||||
while not _stop:
|
||||
connection.process_data_events(time_limit=1)
|
||||
connection.close()
|
||||
except Exception as exc:
|
||||
log.warning("broker.reconnect", extra={"event_fields": json.dumps({"error": type(exc).__name__})})
|
||||
time.sleep(3)
|
||||
log.info("worker.stopped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
|
||||
result = []
|
||||
seen = set()
|
||||
for _ in range(1000):
|
||||
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen,'cursor cycle')
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['persistence','failover'], required=True)
|
||||
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
|
||||
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
|
||||
check(LAB>=2,'persistence requires lab2+')
|
||||
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=10)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
target = sent['headers'].get('X-Instance-Id','')
|
||||
attachment = content = None
|
||||
if LAB>=4:
|
||||
attachment,content = c.attachment(alice,chat)
|
||||
if LAB>=5:
|
||||
attachment = c.wait_ready(alice,attachment)
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
observed = set()
|
||||
if args.mode=='failover':
|
||||
for _ in range(40):
|
||||
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||
# No shell interpolation and no Docker commands in this test.
|
||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
started = time.monotonic()
|
||||
successes,failures,after_instances = [],0,[]
|
||||
try:
|
||||
if args.mode=='persistence':
|
||||
try:
|
||||
code = proc.wait(timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure('restart hook exceeded 90 seconds') from None
|
||||
check(code==0,'restart hook failed (inspect your script locally)')
|
||||
deadline = time.monotonic()+90
|
||||
while True:
|
||||
try:
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
first_recovery = None
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
while time.monotonic()-started < 15:
|
||||
code = proc.poll()
|
||||
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
|
||||
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
|
||||
try:
|
||||
response = c.message(alice,chat,text,key=key)
|
||||
msg = response['json']
|
||||
if code==0:
|
||||
if first_recovery is None:
|
||||
first_recovery = time.monotonic()-started
|
||||
after_instances.append(response['headers']['X-Instance-Id'])
|
||||
successes.append(msg)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
except Failure:
|
||||
failures += 1
|
||||
# Keep SAME key/body after an uncertain POST result.
|
||||
time.sleep(0.1)
|
||||
check(proc.poll()==0,'stop-one hook did not finish successfully')
|
||||
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
|
||||
check(len(after_instances)>=5,'too few successful writes after stop')
|
||||
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
|
||||
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
|
||||
# Existing token, no re-login. Read all confirmations and reject duplicates.
|
||||
messages = history(c,bob,chat)
|
||||
ids = [m['id'] for m in messages]
|
||||
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
|
||||
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
|
||||
for msg in successes:
|
||||
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
|
||||
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
|
||||
texts = [m['text'] for m in messages]
|
||||
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
|
||||
if attachment:
|
||||
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
|
||||
check(downloaded['bytes']==content,'attachment lost or changed across failure')
|
||||
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user