1 Commits
67 changed files with 3169 additions and 87 deletions
+29 -28
View File
@@ -1,41 +1,42 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Credentials for PostgreSQL/S3/broker are generated by the operator into
# ./secrets and injected into OpenBao by scripts/openbao_bootstrap.py.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SERVICE_VERSION=7.0.0
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# lab7: TLS / OpenBao
PKI_DIR=/pki
OPENBAO_ADDR=https://openbao:8200
OPENBAO_CA=/pki/ca.crt
OPENBAO_ROLE_ID_FILE=/run/secrets/openbao_role_id
OPENBAO_SECRET_ID_FILE=/run/secrets/openbao_secret_id
S3_ENDPOINT=https://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_CA=/pki/ca.crt
BROKER_CA=/pki/ca.crt
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
LOGIN_RATE_LIMIT=5
LOGIN_RATE_WINDOW_SECONDS=60
# lab5: durable broker + worker (broker credentials live in OpenBao)
BROKER_USER=change-me
BROKER_PASSWORD=change-me
JOB_QUEUE=attachment_jobs
STAGING_PREFIX=staging
MAX_JOB_ATTEMPTS=5
THUMBNAIL_SIZE=256
THUMBNAIL_QUALITY=85
# lab6: observability over TLS
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4317
OTEL_CA=/pki/ca.crt
OTEL_METRIC_INTERVAL_MS=10000
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
PUBLIC_BASE_URL=https://localhost:8443
TLS_CA_FILE=/run/secrets/ca.crt
TLS_CERT_FILE=/run/secrets/service.crt
TLS_KEY_FILE=/run/secrets/service.key
BAO_ADDR=https://openbao:8200
BAO_CACERT=/run/secrets/bao-ca.crt
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
ARGON2_MEMORY_KIB=19456
ARGON2_ITERATIONS=2
ARGON2_PARALLELISM=1
GRAFANA_ADMIN_PASSWORD=change-me
+23 -14
View File
@@ -1,18 +1,27 @@
## Лабораторная 7
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Лесковский Сергей БПИ2304
Целевая оценка (3/4/5): 3
Трек 5, если заявлен: Не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено:
Весь минимум lab6 сохранён, внешний вход переведён на HTTPS (8443, локальный CA с SAN). Добавлено: обязательный mTLS LB→API через TLS-шлюзы с allowlist CN=lb (чужой CA/identity отклоняются), TLS с проверкой сервера для PostgreSQL (verify-full, hostnossl запрещён), MinIO (verify CA) и RabbitMQ (amqps 5671), TLS OTLP в Collector; OpenBao в production-режиме (file storage, init/unseal оператором, policy read-only на kv/messenger, AppRole token TTL 1h/secret_id 24h, без root token в приложении); Argon2id m=19456,t=2,p=1; login rate limit 429+Retry-After; non-root/read_only/cap_drop/no-new-privileges/лимиты для api и worker; заполнена contracts/trust-matrix.md.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Команды запуска из чистого clone:
См. REPORT.md → «Запуск из чистого clone». Кратко: секреты и .env; bash scripts/pki.sh; docker compose up -d openbao; python scripts/openbao_bootstrap.py; docker compose up -d; python tests/smoke.py --base-url https://localhost:8443 --ca-file pki/ca.crt; python scripts/security_internal.py; python scripts/security_rate_limit.py.
Результаты проверок и ссылка на REPORT.md:
Smoke по HTTPS PASS; security.py PASS (чужой CA отклонён); mTLS: без серта 400, чужой CA 400, CN=other 403, CN=lb 200; OpenBao: свой path 200, чужой 403; rate limit 5×401→429 Retry-After 60; persistence PASS; failover PASS (recovery 3.141 s). Доказательства — evidence/smoke.txt, security-external.txt, security-internal.txt, argon2.txt, rate-limit.txt, hardening.txt, persistence.txt, failover-1.txt, traces.json.
Ограничения / известные ошибки:
Один DB/S3/broker identity для api и worker; mTLS к broker/Collector и TLS для telemetry-backends/Grafana→datasources оставлены в доверенной сети (уровень 4); ротация/revocation, rehash, image scan — уровень 4. OpenBao init/unseal — ручной операторский шаг, после рестарта нужен повторный bootstrap. Один Docker-host, HA нет.
Источники и использование ИИ:
OpenBao 2.0.2, nginx 1.27.1, PostgreSQL 16.4, MinIO, RabbitMQ 3.13, OTel Collector 0.115.1, Prometheus/Loki/Tempo/Grafana, FastAPI 0.115, argon2-cffi, pika 1.3.2, boto3 1.35. ИИ использовался для консультаций по AppRole и nginx mTLS.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+5
View File
@@ -30,3 +30,8 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
# lab7 local PKI material
pki/
certs/
*.srl
+39
View File
@@ -0,0 +1,39 @@
# syntax=docker/dockerfile:1.7
FROM python:3.12-slim AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=60 \
PIP_RETRIES=2
WORKDIR /app
COPY requirements.txt .
# Пробуем несколько зеркал по очереди. Первое, которое ответит, используется.
RUN --mount=type=cache,target=/root/.cache/pip \
set -e; \
for idx in \
"https://pypi.depkit.ru/simple/" \
"https://pypi-mirror.gitverse.ru/simple" \
"https://pypi.org/simple/" \
; do \
echo "==> Trying index: $idx"; \
if pip install --no-cache-dir --index-url "$idx" -r requirements.txt; then \
echo "==> OK: $idx"; \
exit 0; \
fi; \
echo "==> FAILED: $idx"; \
done; \
echo "==> All indexes failed"; \
exit 1
COPY src ./src
RUN useradd -r -u 10001 -s /usr/sbin/nologin appuser
USER appuser
EXPOSE 8080
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
+48 -16
View File
@@ -1,37 +1,69 @@
# Отчёт — лабораторная 7
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- **Автор, группа:** Лесковский Сергей БПИ2304
- **Целевая оценка:** 3; трек 5 не заявлен
- **Commit/PR:** ветка `lab7-final` форка `Leskovskiy/mtusi2026lab7-Leskovskiy`, PR в `mtusi2026/mtusi2026lab7`
- **Использованные источники, библиотеки и помощь ИИ:** OpenBao 2.0.2, nginx 1.27.1 (TLS/mTLS), PostgreSQL 16.4 (TLS), MinIO (TLS), RabbitMQ 3.13 (TLS), OTel Collector 0.115.1 (TLS receiver), Prometheus/Loki/Tempo/Grafana, FastAPI 0.115, Argon2id (argon2-cffi), pika 1.3.2, boto3 1.35. ИИ использовался для консультаций по OpenBao AppRole и конфигурации nginx mTLS.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требования: Docker Compose v2, Python 3, Git Bash (openssl и обёртки на Windows).
1. Сгенерировать локальные секреты: `secrets/postgres_password.txt`, `secrets/s3_root_user.txt`, `secrets/s3_root_password.txt`, `secrets/s3_access_key_id.txt`, `secrets/s3_secret_access_key.txt`, `secrets/broker_user.txt`, `secrets/broker_password.txt`; `.env` по `.env.example` (`BROKER_USER`, `BROKER_PASSWORD`, `GRAFANA_ADMIN_PASSWORD`).
2. `bash scripts/pki.sh` — локальный CA и leaf-сертификаты в `pki/` (в Git не попадают).
3. `docker compose --env-file .env up -d openbao`
4. `python scripts/openbao_bootstrap.py` — init (однократно), unseal, KV, policy, AppRole; самопроверка login/policy.
5. `docker compose --env-file .env up -d`
6. `curl --cacert pki/ca.crt https://localhost:8443/health/ready`
7. Тесты: `python tests/smoke.py --base-url https://localhost:8443 --ca-file pki/ca.crt`; `tests/security.py`; `python scripts/security_internal.py`; `python scripts/security_rate_limit.py`.
8. Остановка: `docker compose --env-file .env down`; удаление данных: `down -v` (плюс `secrets/openbao_init.json` и `pki/` удаляются вручную).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
Клиент --TLS--> lb:8443 --mTLS--> api-gw1/2:8443 --закрытый hop--> api1/api2:8080
(CN=lb allowlist) │
api/worker --TLS--> PostgreSQL, MinIO, RabbitMQ, OTel Collector, OpenBao
```
- **Внешний TLS:** nginx LB, сертификат с SAN `localhost`/`127.0.0.1`; единственный публичный порт — 8443.
- **LB → API:** обязательный mTLS. LB предъявляет сертификат с CN=lb; шлюз проверяет цепочку (`ssl_verify_client on`) и identity (allowlist CN=lb); чужой CA, отсутствие сертификата и CN≠lb отклоняются. Шлюз → API — закрытый compose-hop без публикации портов.
- **Хранилища:** PostgreSQL с `ssl=on` и `pg_hba`, запрещающим `hostnossl`; клиент `sslmode=verify-full`. MinIO с TLS (`--certs-dir`), boto3 `verify=CA`. RabbitMQ с отключённым plain-TCP и TLS-listener 5671, pika проверяет hostname.
- **Телеметрия:** OTel Collector принимает OTLP по TLS; SDK проверяет CA. Backends (Prometheus/Loki/Tempo/Grafana) — внутри доверенной сети, наружу опубликованы только `127.0.0.1` admin-интерфейсы.
- **OpenBao:** production-режим (file storage + TLS), init/unseal оператором, policy `messenger` (read только `kv/data/messenger`), AppRole с token TTL 1h/secret_id TTL 24h. Приложение получает `database_url`, S3-ключи и broker URL через AppRole; root token и unseal key в приложении не используются.
- **Argon2id:** `time_cost=2, memory_cost=19456 (19 MiB), parallelism=1, hash_len=32, salt_len=16`; измеренное время verify ~24.5 ms (evidence/argon2.txt).
- **Лимиты:** login rate limit (5 попыток/60 с на username+IP, общий через PostgreSQL) → 429 + `Retry-After: 60`; размер запроса ограничен nginx (11 MiB) и API (10 MiB).
- **Контейнеры:** api/worker — non-root (`appuser`), `read_only`, `cap_drop: ALL`, `no-new-privileges`, tmpfs `/tmp`, memory 512m, 1 CPU (evidence/hardening.txt).
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Предыдущие возможности через HTTPS | `tests/smoke.py --ca-file pki/ca.crt` | `PASS: lab7 public API scenario` | evidence/smoke.txt |
| Чужой CA отклоняется | `tests/security.py` | `PASS: trusted HTTPS works; untrusted CA rejected` | evidence/security-external.txt |
| LB→API mTLS: нет серта/чужой CA/чужая identity | `scripts/security_internal.py` | 400 / 400 / 403 | evidence/security-internal.txt |
| LB identity допускается | `scripts/security_internal.py` | 200 | evidence/security-internal.txt |
| OpenBao scoped policy | bootstrap self-check + `security_internal.py` | свой path 200, чужой 403 | evidence/security-internal.txt |
| Argon2id параметры | `docker compose exec api1 python …` | `$argon2id$v=19$m=19456,t=2,p=1`, verify ~24.5 ms | evidence/argon2.txt |
| Login rate limit | `scripts/security_rate_limit.py` | 5×401, затем 429 + `Retry-After: 60` | evidence/rate-limit.txt |
| Хардненинг контейнеров | `docker inspect` | ro, cap_drop ALL, no-new-privileges, non-root, limits | evidence/hardening.txt |
| Сохранность и failover через HTTPS | `tests/resilience*.py --ca-file` | persistence PASS; failover PASS (recovery 3.141 s) | evidence/persistence.txt, failover-1.txt |
| Телеметрия после TLS | Prometheus/Tempo query | метрики и трассы присутствуют | evidence/traces.json |
| Матрица доверия заполнена | `contracts/trust-matrix.md` | все связи, identity, тесты | contracts/trust-matrix.md |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Рестарт stores + OpenBao + API (persistence hook) | Сессии/данные сохранены, OpenBao разsealed оператором | `PASS: persistence preserved...` | потерь нет |
| Остановка одной API-реплики | LB уходит на вторую, mTLS сохраняется | 12 записей, recovery 3.141 s, 9 неудачных попыток в окне | потерь подтверждённых сообщений нет |
| Остановка/рестарт OpenBao | API не стартует, пока OpenBao sealed; после unseal восстанавливается | bootstrap → API healthy | простой на время unseal |
| Недоверенный клиентский сертификат | TLS/HTTP отказ | 400/403 | — |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Раздельные DB/S3/broker identity для api и worker не реализованы (один пользователь на сервис); mTLS к broker/Collector, TLS для telemetry-backends и Grafana→datasources оставлены внутри доверенной сети — это уровень 4. Для продакшена нужны отдельные роли, scoped S3-политики и TLS/mTLS на всех внутренних hop.
- Ротация/expiry/revocation короткоживущих сертификатов и SecretID, rehash пароля, dependency/image scan, read-only для stateful-сервисов — уровень 4.
- OpenBao init/unseal — ручной операторский шаг (разрешённый компромисс); после рестарта требуется повторный bootstrap.
- Один Docker-host; HA хранилищ и OpenBao не заявляется.
- Argon2id параметры обоснованы измерением на этом стенде (~24.5 ms verify), memory 19 MiB соответствует рекомендации OWASP для интерактивного логина.
+312
View File
@@ -0,0 +1,312 @@
name: lab7
x-api-env: &api_env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: lab
SERVICE_VERSION: "7.0.0"
SESSION_TTL_SECONDS: "3600"
PKI_DIR: /pki
OPENBAO_ADDR: https://openbao:8200
OPENBAO_CA: /pki/ca.crt
OPENBAO_ROLE_ID_FILE: /run/secrets/openbao_role_id
OPENBAO_SECRET_ID_FILE: /run/secrets/openbao_secret_id
S3_ENDPOINT: https://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_CA: /pki/ca.crt
BROKER_CA: /pki/ca.crt
JOB_QUEUE: attachment_jobs
STAGING_PREFIX: staging
MAX_JOB_ATTEMPTS: "5"
THUMBNAIL_SIZE: "256"
MAX_UPLOAD_BYTES: "10485760"
MAX_IMAGE_PIXELS: "40000000"
LOGIN_RATE_LIMIT: "5"
LOGIN_RATE_WINDOW_SECONDS: "60"
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4317
OTEL_CA: /pki/ca.crt
OTEL_METRIC_INTERVAL_MS: "10000"
x-hardening: &hardening
read_only: true
tmpfs:
- /tmp
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
mem_limit: 512m
cpus: 1.0
services:
postgres:
image: postgres:16.4-alpine
command:
- sh
- -c
- >-
install -m 0600 -o postgres /pki/postgres.key /tmp/postgres.key &&
install -m 0644 /pki/postgres.crt /tmp/postgres.crt &&
install -m 0644 /pki/ca.crt /tmp/ca.crt &&
exec docker-entrypoint.sh postgres
-c ssl=on
-c ssl_cert_file=/tmp/postgres.crt
-c ssl_key_file=/tmp/postgres.key
-c ssl_ca_file=/tmp/ca.crt
-c hba_file=/etc/postgresql/pg_hba.conf
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_DB: messenger
volumes:
- pgdata:/var/lib/postgresql/data
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
- ./pki:/pki:ro
secrets:
- postgres_password
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 2s
retries: 30
s3:
image: milvusdb/minio:RELEASE.2024-12-18T13-15-44Z
command: server /data --certs-dir /certs --console-address ":9001"
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
volumes:
- s3data:/data
- ./pki:/pki:ro
- ./pki/minio.crt:/certs/public.crt:ro
- ./pki/minio.key:/certs/private.key:ro
secrets:
- s3_root_user
- s3_root_password
healthcheck:
test: ["CMD-SHELL", "curl -fsS --cacert /pki/ca.crt https://localhost:9000/minio/health/live || exit 1"]
interval: 5s
timeout: 3s
retries: 30
broker:
image: rabbitmq:3.13-alpine
environment:
RABBITMQ_DEFAULT_USER: ${BROKER_USER:?set BROKER_USER in .env}
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?set BROKER_PASSWORD in .env}
volumes:
- rabbitdata:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
- ./pki:/pki:ro
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 30
openbao-perms:
image: openbao/openbao:2.0.2
user: "0:0"
entrypoint: ["sh", "-c", "chown -R 100:1000 /openbao/data"]
volumes:
- openbaodata:/openbao/data
openbao:
image: openbao/openbao:2.0.2
command: ["server", "-config=/openbao/config.hcl"]
environment:
BAO_ADDR: https://openbao:8200
BAO_CACERT: /pki/ca.crt
cap_add:
- IPC_LOCK
ports:
- "127.0.0.1:8200:8200"
volumes:
- openbaodata:/openbao/data
- ./infra/openbao/config.hcl:/openbao/config.hcl:ro
- ./pki:/pki:ro
depends_on:
openbao-perms:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "BAO_ADDR=https://openbao:8200 BAO_CACERT=/pki/ca.crt bao status >/dev/null 2>&1"]
interval: 5s
timeout: 3s
retries: 30
api1:
build: .
image: lab7-api:local
<<: *hardening
environment:
<<: *api_env
INSTANCE_ID: api-1
secrets:
- openbao_role_id
- openbao_secret_id
volumes:
- ./pki:/pki:ro
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
openbao:
condition: service_healthy
api2:
image: lab7-api:local
<<: *hardening
environment:
<<: *api_env
INSTANCE_ID: api-2
secrets:
- openbao_role_id
- openbao_secret_id
volumes:
- ./pki:/pki:ro
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
openbao:
condition: service_healthy
worker:
image: lab7-api:local
<<: *hardening
command: ["python", "-m", "src.worker"]
environment:
<<: *api_env
INSTANCE_ID: worker
secrets:
- openbao_role_id
- openbao_secret_id
volumes:
- ./pki:/pki:ro
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
broker:
condition: service_healthy
openbao:
condition: service_healthy
api-gw1:
image: nginx:1.27.1-alpine
environment:
API_UPSTREAM: api1
volumes:
- ./infra/nginx/gateway.conf.template:/etc/nginx/templates/gateway.conf.template:ro
- ./pki:/pki:ro
ports:
- "127.0.0.1:9443:8443"
depends_on:
- api1
api-gw2:
image: nginx:1.27.1-alpine
environment:
API_UPSTREAM: api2
volumes:
- ./infra/nginx/gateway.conf.template:/etc/nginx/templates/gateway.conf.template:ro
- ./pki:/pki:ro
ports:
- "127.0.0.1:9444:8443"
depends_on:
- api2
lb:
image: nginx:1.27.1-alpine
ports:
- "8443:8443"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./pki:/pki:ro
depends_on:
- api-gw1
- api-gw2
otel-collector:
image: otel/opentelemetry-collector-contrib:0.115.1
command: ["--config=/etc/otelcol/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
- ./pki:/pki:ro
prometheus:
image: prom/prometheus:v2.55.1
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- promdata:/prometheus
ports:
- "127.0.0.1:9090:9090"
depends_on:
- otel-collector
loki:
image: grafana/loki:3.2.1
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
tempo:
image: grafana/tempo:2.6.1
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
grafana:
image: grafana/grafana:11.3.1
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
GF_USERS_DEFAULT_THEME: dark
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
depends_on:
- prometheus
- loki
- tempo
volumes:
pgdata: {}
s3data: {}
rabbitdata: {}
openbaodata: {}
promdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
secrets:
postgres_password:
file: ./secrets/postgres_password.txt
s3_root_user:
file: ./secrets/s3_root_user.txt
s3_root_password:
file: ./secrets/s3_root_password.txt
openbao_role_id:
file: ./secrets/openbao_role_id.txt
openbao_secret_id:
file: ./secrets/openbao_secret_id.txt
+25 -29
View File
@@ -1,37 +1,33 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия — lab7 (реализация)
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
Локальный CA `pki/ca.crt` создаётся оператором через `scripts/pki.sh` (не коммитится). Каждый leaf имеет CN и SAN; серверы проверяют цепочку и hostname, mTLS-шлюзы дополнительно проверяют identity клиента.
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
| Связь | Реализация (порт/endpoint) | CA / identity / проверка | Срок credential | Тест |
| --- | --- | --- | --- | --- |
| Клиент → ingress/LB | `https://localhost:8443` (nginx, TLS 1.2/1.3) | `ca.crt`, SAN `DNS:localhost,IP:127.0.0.1`; hostname проверяется клиентом; Bearer-сессия | сертификат 825 дней; сессия `SESSION_TTL_SECONDS` | `tests/security.py` (чужой CA → ошибка верификации); `tests/smoke.py` |
| LB → API (все реплики) | LB → `api-gw1/api-gw2:8443` (mTLS), шлюз → API `:8080` по закрытому compose-hop | серверный серт `gateway` (SAN api-gw1/api-gw2), клиентский серт `lb` (CN=lb); `ssl_verify_client on`, allowlist `CN=lb`, чужой CA/identity отклоняются | leaf 825 дней; CN allowlist | `scripts/security_internal.py`: без серта → 400, чужой CA → 400, CN=other → 403, CN=lb → 200 |
| API/worker → PostgreSQL | `postgres:5432`, `sslmode=verify-full` + `sslrootcert=/pki/ca.crt` | CA + SAN `postgres`; `pg_hba` запрещает `hostnossl`; роль `messenger` с правами только на свою БД | пароль из OpenBao | smoke + persistence (подключение падает при неверном CA) |
| API/worker → S3 (MinIO) | `https://s3:9000`, boto3 `verify=/pki/ca.crt` | CA + SAN `s3`; bucket приватный; ключи из OpenBao | ключи из OpenBao | smoke (upload/download) |
| API/worker → broker | `amqps://broker:5671` (TLS listener, plain TCP listener отключён) | CA + SAN `broker`; pika `SSLOptions` с проверкой hostname | пароль из OpenBao | smoke (job → ready), failover |
| API/worker → Collector | `https://otel-collector:4317` (OTLP/gRPC + TLS) | CA + SAN `otel-collector`; gRPC `ssl_channel_credentials` | — | метрики/трассы/логи в Grafana |
| Collector → telemetry backends | внутри доверенной сети compose (Tempo/Loki/Prometheus) | **исключение**: сеть стенда считается доверенной зоной; публичных портов у backends нет | — | отчёт, ограничения |
| Grafana → data sources | внутри доверенной сети, порт `127.0.0.1:3000` | provisioning datasources; наружу не публикуется | — | Grafana health |
| API/worker → OpenBao | `https://openbao:8200`, AppRole login → token, KV v2 read `kv/data/messenger` | CA + SAN `openbao`; policy `messenger` разрешает только чтение своего path; root token приложению недоступен | SecretID 24h, token TTL 1h, max 4h | `scripts/security_internal.py`: свой path 200, чужой 403; bootstrap self-check |
| Оператор → OpenBao/Grafana/Prometheus/S3 admin | `127.0.0.1:8200/3000/9090`, admin API MinIO не публикуется | OpenBao TLS; root token/unseal key только в `secrets/openbao_init.json` у оператора | — | bootstrap, отчёт |
| API → Redis/Valkey | N/A | не используется (сессии в PostgreSQL) | — | — |
| Replication/кластерные каналы | N/A | одиночные узлы | — | — |
## Обязательные отрицательные проверки
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
1. HTTPS с правильным CA/hostname проходит; с чужим CA и неверным hostname — ошибка верификации: `tests/security.py` PASS.
2. LB→API: без клиентского серта / чужой CA / запрещённая identity → отказ; CN=lb → успех: `scripts/security_internal.py` PASS.
3. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; `X-User-Id` не подменяет Bearer: публичный smoke PASS.
4. OpenBao app policy читает только `kv/messenger`, запись и чужой path запрещены: self-check и `scripts/security_internal.py` PASS.
5. Login rate limit: 5 неудач → 429 + `Retry-After`: `scripts/security_rate_limit.py` PASS.
## Bootstrap и эксплуатация
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
- `scripts/pki.sh` создаёт CA и leaf-сертификаты в `pki/` (gitignored). CA key и leaf keys остаются у оператора.
- `scripts/openbao_bootstrap.py` (после `docker compose up -d openbao`) выполняет init (однократно), unseal, включение `kv` v2, запись runtime-секретов, policy, AppRole; root token и unseal key сохраняет в `secrets/openbao_init.json` (gitignored).
- После рестарта OpenBao он снова sealed: оператор повторяет bootstrap (идемпотентно). Это же делает `scripts/restart.sh` в persistence-сценарии.
- Приложение получает секреты только через AppRole; root token, unseal keys и SecretID не попадают в образы, Git и телеметрию.
Binary file not shown.
+16
View File
@@ -0,0 +1,16 @@
SERVICE STATUS PORTS
api-gw1 Up About a minute 127.0.0.1:9443->8443/tcp
api-gw2 Up About a minute 127.0.0.1:9444->8443/tcp
api1 Up About a minute 8080/tcp
api2 Up About a minute 8080/tcp
broker Up 3 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
grafana Up 6 minutes 127.0.0.1:3000->3000/tcp
lb Up About a minute 0.0.0.0:8443->8443/tcp, [::]:8443->8443/tcp
loki Up 6 minutes 3100/tcp
openbao Up 3 minutes (healthy) 127.0.0.1:8200->8200/tcp
otel-collector Up 6 minutes 4317-4318/tcp, 55678-55679/tcp
postgres Up 3 minutes (healthy) 5432/tcp
prometheus Up 6 minutes 127.0.0.1:9090->9090/tcp
s3 Up 3 minutes (healthy) 9000/tcp
tempo Up 6 minutes
worker Up 3 minutes 8080/tcp
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
{"traces":[{"traceID":"10e1b3be92e1a99f516fb72cb89d5da8","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/messages","startTimeUnixNano":"1795110028472495804","durationMs":8},{"traceID":"1c58ef73eeca445458f3d8b66f0c7669","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats","startTimeUnixNano":"1795110028052560529","durationMs":22}],"metrics":{"inspectedTraces":38,"inspectedBytes":"20488","completedJobs":1,"totalJobs":1}}
+108
View File
@@ -0,0 +1,108 @@
{
"uid": "messenger-lab6",
"title": "Messenger — lab6 observability",
"tags": ["messenger", "lab6"],
"schemaVersion": 39,
"version": 1,
"refresh": "10s",
"editable": true,
"time": { "from": "now-30m", "to": "now" },
"timezone": "utc",
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "API RPS (excluding health)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m]))",
"legendFormat": "RPS"
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "API 5xx ratio",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum(rate(http_server_requests_total{http_response_status_code=~\"5..\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])), 0.001)",
"legendFormat": "5xx / business"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "HTTP duration p95",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
"legendFormat": "p95"
},
{
"refId": "B",
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
"legendFormat": "p50"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Attachment jobs by outcome",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "sum by (job_outcome) (rate(attachment_jobs_total[1m]))",
"legendFormat": "{{job_outcome}}"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Worker job duration p95 by outcome",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, job_outcome) (rate(attachment_job_duration_seconds_bucket[5m])))",
"legendFormat": "{{job_outcome}}"
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "API request rate by route (top 10)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"expr": "topk(10, sum by (http_route, http_request_method) (rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
}
]
}
@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
updateIntervalSeconds: 30
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,28 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-1m"
spanEndTimeShift: "1m"
filterByTraceID: true
filterBySpanID: false
+26
View File
@@ -0,0 +1,26 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
+30
View File
@@ -0,0 +1,30 @@
map $ssl_client_s_dn $lb_identity_ok {
default 0;
"~*CN=lb(,|/|$)" 1;
}
server {
listen 8443 ssl;
server_name api-gateway;
ssl_certificate /pki/gateway.crt;
ssl_certificate_key /pki/gateway.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_client_certificate /pki/ca.crt;
ssl_verify_client on;
ssl_verify_depth 2;
# A trusted CA is not enough: only the LB identity may call the API.
if ($lb_identity_ok = 0) {
return 403;
}
location / {
proxy_pass http://${API_UPSTREAM}:8080;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Client-Identity $ssl_client_s_dn;
proxy_connect_timeout 2s;
proxy_read_timeout 30s;
}
}
+41
View File
@@ -0,0 +1,41 @@
events {}
http {
upstream api_secure {
server api-gw1:8443 max_fails=1 fail_timeout=2s;
server api-gw2:8443 max_fails=1 fail_timeout=2s;
}
server {
listen 8443 ssl;
server_name localhost;
ssl_certificate /pki/server.crt;
ssl_certificate_key /pki/server.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_cache shared:SSL:5m;
client_max_body_size 11m;
location / {
proxy_pass https://api_secure;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 3s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
# A stopped replica makes its gateway answer 502; retry the other
# gateway. Message POSTs carry Idempotency-Key, so a retry is safe.
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 3s;
# Mandatory mTLS to every API replica: verify the gateway chain and
# present the LB identity certificate.
proxy_ssl_verify on;
proxy_ssl_trusted_certificate /pki/ca.crt;
proxy_ssl_certificate /pki/lb.crt;
proxy_ssl_certificate_key /pki/lb.key;
proxy_ssl_server_name on;
proxy_ssl_name api-gw1;
}
}
}
+17
View File
@@ -0,0 +1,17 @@
ui = false
disable_mlock = true
storage "file" {
path = "/openbao/data"
}
listener "tcp" {
address = "0.0.0.0:8200"
tls_disable = 0
tls_cert_file = "/pki/openbao.crt"
tls_key_file = "/pki/openbao.key"
tls_min_version = "tls12"
}
api_addr = "https://openbao:8200"
cluster_addr = "https://openbao:8201"
+8
View File
@@ -0,0 +1,8 @@
# Application policy: read only the KV v2 data and metadata of its own path.
path "kv/data/messenger" {
capabilities = ["read"]
}
path "kv/metadata/messenger" {
capabilities = ["read"]
}
+53
View File
@@ -0,0 +1,53 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
tls:
cert_file: /pki/otel.crt
key_file: /pki/otel.key
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: /pki/otel.crt
key_file: /pki/otel.key
processors:
memory_limiter:
check_interval: 1s
limit_mib: 256
spike_limit_mib: 64
batch:
timeout: 5s
send_batch_size: 1024
send_batch_max_size: 2048
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: false
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
loki:
endpoint: http://loki:3100/loki/api/v1/push
service:
telemetry:
logs:
level: warn
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [loki]
+4
View File
@@ -0,0 +1,4 @@
# Require TLS for every TCP connection; no plaintext host entry is accepted.
local all all trust
hostssl all all all scram-sha-256
hostnossl all all all reject
+8
View File
@@ -0,0 +1,8 @@
global:
scrape_interval: 5s
evaluation_interval: 15s
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8889"]
+10
View File
@@ -0,0 +1,10 @@
listeners.tcp = none
listeners.ssl.default = 5671
ssl_options.cacertfile = /pki/ca.crt
ssl_options.certfile = /pki/broker.crt
ssl_options.keyfile = /pki/broker.key
ssl_options.verify = verify_none
ssl_options.fail_if_no_peer_cert = false
loopback_users = none
+19
View File
@@ -0,0 +1,19 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
local:
path: /var/tempo/traces
wal:
path: /var/tempo/wal
+12
View File
@@ -0,0 +1,12 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
sqlalchemy==2.0.35
psycopg[binary]==3.2.3
boto3==1.35.36
argon2-cffi==23.1.0
pillow==10.4.0
pydantic==2.9.2
pika==1.3.2
opentelemetry-sdk==1.28.2
opentelemetry-exporter-otlp==1.28.2
opentelemetry-instrumentation-fastapi==0.49b2
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""lab5 failure drills: broker restart durability and bounded retry.
A) Stop the worker, accept an upload, restart the broker (persistent volume),
start the worker: the accepted job still becomes ready.
B) Accept an upload while the worker is down, stop S3, start the worker:
transient failures exhaust MAX_JOB_ATTEMPTS and the attachment ends as
failed with a safe error_code. S3 is restored afterwards.
"""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*args):
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_status(c, actor, att, wanted, seconds=120):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
row = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
if row["status"] in wanted:
return row
time.sleep(0.5)
raise AssertionError("attachment did not reach " + repr(wanted))
def main():
args = arguments("lab5 failure drills")
c = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, _eve, chat = c.bootstrap()
print("A. broker restart keeps accepted work")
compose("stop", "worker")
try:
att, _content = c.attachment(alice, chat)
compose("restart", "broker")
deadline = time.monotonic() + 90
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "API did not recover after broker restart")
time.sleep(0.5)
compose("up", "-d", "--no-deps", "worker")
started = time.monotonic()
ready = c.wait_ready(bob, att, wait_seconds=90)
print(json.dumps({
"scenario": "broker_restart",
"attachment_id": att["id"],
"status": ready["status"],
"seconds_to_ready": round(time.monotonic() - started, 3),
}, ensure_ascii=False))
check(ready["status"] == "ready", "job lost after broker restart")
finally:
compose("up", "-d", "--no-deps", "worker")
print("B. bounded retry ends in a safe failed state")
compose("stop", "worker")
try:
att, _content = c.attachment(alice, chat)
compose("stop", "s3")
compose("up", "-d", "--no-deps", "worker")
failed = wait_status(c, bob, att, {"failed"}, seconds=180)
print(json.dumps({
"scenario": "retry_exhausted",
"attachment_id": att["id"],
"status": failed["status"],
"error_code": failed["error_code"],
}, ensure_ascii=False))
check(failed["error_code"] in ("processing_failed", "invalid_image"), "unsafe error_code")
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
finally:
compose("up", "-d", "s3")
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 60
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "S3 did not recover")
time.sleep(0.5)
print("PASS: broker durability and bounded retry demonstrated.")
if __name__ == "__main__":
run(main)
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""Demonstrate the lab5 acceptance scenario without hiding infrastructure steps.
Worker stopped -> upload returns 202 queued -> message references the chat
attachment -> API replicas restarted -> worker started -> the same job
reaches ready with a contract-compliant JPEG thumbnail.
The script prints only synthetic IDs and timings. It never prints tokens.
"""
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*args):
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def main():
args = arguments("lab5 worker-outage demonstration")
c = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, _eve, chat = c.bootstrap()
print("1. stop worker")
compose("stop", "worker")
try:
print("2. upload with worker down")
att, content = c.attachment(alice, chat)
check(att["status"] == "queued", "queued expected after upload")
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
check(meta["status"] in ("queued", "processing"), "still unprocessed while worker is down")
pending = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
check(pending["status"] == 409, "content must be 409 while queued")
print("3. attach the queued attachment to a message")
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
check(msg["attachment_ids"] == [att["id"]], "message lost the attachment")
print("4. restart API replicas (accepted work must survive)")
compose("restart", "api1", "api2")
deadline = time.monotonic() + 90
while True:
try:
c.call("GET", "/health/ready", schema="Health")
break
except Exception:
check(time.monotonic() < deadline, "API did not recover")
time.sleep(0.5)
print("5. start worker")
compose("up", "-d", "--no-deps", "worker")
started = time.monotonic()
ready = c.wait_ready(bob, att, wait_seconds=90)
elapsed = round(time.monotonic() - started, 3)
check(ready["status"] == "ready", "job did not reach ready")
variants = [v for v in ready["variants"] if v["name"] == "thumbnail"]
check(len(variants) == 1, "expected exactly one thumbnail")
v = variants[0]
check((v["width"], v["height"]) == (256, 160), "thumbnail dimensions differ")
thumb = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob)
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be JPEG")
from image_probe import jpeg_size
check(jpeg_size(thumb["bytes"]) == (256, 160), "actual JPEG size differs")
print(json.dumps({
"attachment_id": att["id"],
"message_id": msg["id"],
"status_after_worker_restart": ready["status"],
"thumbnail": {"width": v["width"], "height": v["height"],
"size_bytes": v["size_bytes"], "sha256": v["sha256"]},
"seconds_from_worker_start_to_ready": elapsed,
}, ensure_ascii=False))
print("PASS: queued job completed after API and worker restart.")
finally:
compose("up", "-d", "--no-deps", "worker")
if __name__ == "__main__":
run(main)
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
"""Operator bootstrap for OpenBao (lab7).
Run AFTER `docker compose up -d openbao` and BEFORE starting the API:
python scripts/openbao_bootstrap.py
It initializes OpenBao (once), unseals it, creates the KV v2 path, writes the
runtime credentials that are already in ./secrets, installs the scoped policy,
enables AppRole and writes role_id/secret_id files for the application.
Operator-held material (unseal key, root token) is written to
./secrets/openbao_init.json and never leaves this machine. The application
never sees it. Delete ./secrets/openbao_init.json only after moving unseal
material to a safe place.
"""
import base64
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ENV = ["docker", "compose", "--env-file", ".env"]
INIT_FILE = ROOT / "secrets" / "openbao_init.json"
def read(name: str) -> str:
return (ROOT / "secrets" / name).read_text().strip()
def bao(*args: str, stdin: str | None = None, token: str | None = None):
command = ENV + ["exec", "-T"]
if token:
command += ["-e", f"BAO_TOKEN={token}"]
command += ["openbao", "bao", *args]
return subprocess.run(command, cwd=ROOT, input=stdin, text=True, capture_output=True)
def wait_for_container() -> None:
for _ in range(60):
probe = bao("status")
if probe.returncode in (0, 2):
return
time.sleep(1)
raise SystemExit("OpenBao container is not reachable: " + probe.stderr.strip())
def status() -> dict:
probe = bao("status", "-format=json")
try:
return json.loads(probe.stdout)
except ValueError:
return {"initialized": False, "sealed": True}
def ensure_initialized() -> tuple[str, str]:
current = status()
if not current.get("initialized", False):
if INIT_FILE.exists():
raise SystemExit("OpenBao is uninitialized but secrets/openbao_init.json exists; "
"refusing to overwrite operator material")
init = bao("operator", "init", "-format=json", "-key-shares=1", "-key-threshold=1")
if init.returncode != 0:
raise SystemExit("operator init failed: " + init.stderr.strip())
data = json.loads(init.stdout)
INIT_FILE.write_text(json.dumps(data, indent=2))
INIT_FILE.chmod(0o600)
print("OpenBao initialized; unseal key and root token saved to secrets/openbao_init.json")
data = json.loads(INIT_FILE.read_text())
key = data["unseal_keys_b64"][0]
token = data["root_token"]
return key, token
def ensure_unsealed(key: str) -> None:
if status().get("sealed", True):
unseal = bao("operator", "unseal", key)
if unseal.returncode != 0:
raise SystemExit("unseal failed: " + unseal.stderr.strip())
print("OpenBao unsealed")
def ensure_kv(token: str) -> None:
mounts = bao("secrets", "list", "-format=json", token=token)
existing = json.loads(mounts.stdout or "{}")
if "kv/" not in existing:
enable = bao("secrets", "enable", "-path=kv", "-version=2", "kv", token=token)
if enable.returncode != 0:
raise SystemExit("kv enable failed: " + enable.stderr.strip())
print("Enabled kv v2 at kv/")
def write_runtime_secrets(token: str) -> None:
pg_password = read("postgres_password.txt")
broker_user = read("broker_user.txt")
broker_password = read("broker_password.txt")
database_url = ("postgresql+psycopg://messenger:" + pg_password +
"@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/pki/ca.crt")
broker_url = f"amqps://{broker_user}:{broker_password}@broker:5671/"
write = bao("kv", "put", "kv/messenger",
f"database_url={database_url}",
f"s3_access_key_id={read('s3_access_key_id.txt')}",
f"s3_secret_access_key={read('s3_secret_access_key.txt')}",
f"broker_url={broker_url}",
token=token)
if write.returncode != 0:
raise SystemExit("kv put failed: " + write.stderr.strip())
print("Wrote kv/messenger")
def install_policy(token: str) -> None:
policy = (ROOT / "infra" / "openbao" / "policy-messenger.hcl").read_text()
write = bao("policy", "write", "messenger", "-", stdin=policy, token=token)
if write.returncode != 0:
raise SystemExit("policy write failed: " + write.stderr.strip())
print("Installed policy messenger")
def ensure_approle(token: str) -> None:
enable = bao("auth", "enable", "approle", token=token)
if enable.returncode != 0 and "path is already in use" not in (enable.stderr or ""):
if "already enabled" not in (enable.stderr or ""):
raise SystemExit("approle enable failed: " + enable.stderr.strip())
role = bao("write", "auth/approle/role/messenger",
"token_ttl=1h", "token_max_ttl=4h", "secret_id_ttl=24h",
"policies=messenger", token=token)
if role.returncode != 0:
raise SystemExit("approle role failed: " + role.stderr.strip())
role_id = bao("read", "-field=role_id", "auth/approle/role/messenger/role-id", token=token)
secret_id = bao("write", "-f", "-field=secret_id", "auth/approle/role/messenger/secret-id", token=token)
if role_id.returncode != 0 or secret_id.returncode != 0:
raise SystemExit("could not issue AppRole credentials")
(ROOT / "secrets" / "openbao_role_id.txt").write_text(role_id.stdout.strip())
(ROOT / "secrets" / "openbao_secret_id.txt").write_text(secret_id.stdout.strip())
print("Issued AppRole role_id/secret_id")
def self_check(role_id: str, secret_id: str) -> None:
login = bao("write", "-format=json", "auth/approle/login",
f"role_id={role_id}", f"secret_id={secret_id}")
if login.returncode != 0:
raise SystemExit("AppRole login self-check failed: " + login.stderr.strip())
token = json.loads(login.stdout)["auth"]["client_token"]
read_back = bao("kv", "get", "-format=json", "kv/messenger", token=token)
if read_back.returncode != 0:
raise SystemExit("policy cannot read kv/messenger: " + read_back.stderr.strip())
forbidden = bao("kv", "get", "-format=json", "kv/other", token=token)
if forbidden.returncode == 0:
raise SystemExit("policy unexpectedly reads kv/other")
print("AppRole login and scoped policy verified (kv/other denied)")
def main() -> None:
wait_for_container()
key, root = ensure_initialized()
ensure_unsealed(key)
ensure_kv(root)
write_runtime_secrets(root)
install_policy(root)
ensure_approle(root)
self_check((ROOT / "secrets" / "openbao_role_id.txt").read_text().strip(),
(ROOT / "secrets" / "openbao_secret_id.txt").read_text().strip())
print("OpenBao bootstrap complete. Start the rest: docker compose up -d")
if __name__ == "__main__":
main()
+78
View File
@@ -0,0 +1,78 @@
#!/bin/sh
# Local lab7 PKI bootstrap. Never commit pki/: the CA key and leaf keys are
# operator-held material. Run from the repository root (Git Bash on Windows).
set -eu
# Git Bash on Windows rewrites arguments that look like Unix paths (e.g. the
# openssl -subj value). Keep the arguments byte-exact.
export MSYS_NO_PATHCONV=1
DAYS=825
PKI=${PKI_DIR:-pki}
mkdir -p "$PKI"
cd "$PKI"
if [ -f ca.crt ]; then
echo "PKI already exists in $PKI; delete it to regenerate."
exit 0
fi
write_ext() {
# $1 file, $2 san
cat > "$1" <<EOF
basicConstraints=CA:FALSE
keyUsage=digitalSignature,keyEncipherment
extendedKeyUsage=serverAuth,clientAuth
subjectAltName=$2
EOF
}
echo "==> CA"
openssl genrsa -out ca.key 4096 >/dev/null 2>&1
openssl req -x509 -new -key ca.key -sha256 -days $DAYS \
-subj "/CN=mtusi-lab7-local-ca" \
-addext "basicConstraints=critical,CA:TRUE" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-out ca.crt >/dev/null 2>&1
issue() {
# $1 name, $2 CN, $3 SAN
openssl genrsa -out "$1.key" 2048 >/dev/null 2>&1
openssl req -new -key "$1.key" -subj "/CN=$2" -out "$1.csr" >/dev/null 2>&1
write_ext "$1.ext" "$3"
openssl x509 -req -in "$1.csr" -CA ca.crt -CAkey ca.key -CAcreateserial \
-days $DAYS -sha256 -extfile "$1.ext" -out "$1.crt" >/dev/null 2>&1
rm -f "$1.csr" "$1.ext"
chmod 600 "$1.key" 2>/dev/null || true
}
echo "==> leaf certificates"
issue server "localhost" "DNS:localhost,IP:127.0.0.1"
issue lb "lb" "DNS:lb"
issue gateway "api-gateway" "DNS:api-gw1,DNS:api-gw2,DNS:localhost"
issue postgres "postgres" "DNS:postgres"
issue minio "s3" "DNS:s3,DNS:minio,DNS:localhost"
issue broker "broker" "DNS:broker"
issue otel "otel-collector" "DNS:otel-collector"
issue openbao "openbao" "DNS:openbao,IP:127.0.0.1"
issue other "other-service" "DNS:other-service"
# A second, untrusted CA and a client certificate signed by it: used by the
# negative mTLS test (valid chain, but the wrong root).
echo "==> foreign CA (negative test only)"
openssl genrsa -out foreign-ca.key 2048 >/dev/null 2>&1
openssl req -x509 -new -key foreign-ca.key -sha256 -days $DAYS \
-subj "/CN=foreign-ca" \
-addext "basicConstraints=critical,CA:TRUE" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-out foreign-ca.crt >/dev/null 2>&1
openssl genrsa -out foreign-client.key 2048 >/dev/null 2>&1
openssl req -new -key foreign-client.key -subj "/CN=foreign-client" -out foreign-client.csr >/dev/null 2>&1
write_ext foreign-client.ext "DNS:foreign-client"
openssl x509 -req -in foreign-client.csr -CA foreign-ca.crt -CAkey foreign-ca.key \
-CAcreateserial -days $DAYS -sha256 -extfile foreign-client.ext -out foreign-client.crt >/dev/null 2>&1
rm -f foreign-client.csr foreign-client.ext
rm -f ca.srl foreign-ca.srl
echo "PKI ready in $PKI/"
ls -1 *.crt
+2
View File
@@ -0,0 +1,2 @@
@echo off
"C:\Program Files\Git\bin\bash.exe" "%~dp0pki.sh"
Executable → Regular
View File
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
# Restart the stores and the OpenBao node. OpenBao comes back sealed, so the
# documented operator step is to re-run the bootstrap (idempotent unseal)
# before the API replicas need their secrets again.
docker compose --env-file .env restart postgres s3 broker openbao
if command -v py >/dev/null 2>&1; then
py -3 scripts/openbao_bootstrap.py
else
python scripts/openbao_bootstrap.py
fi
docker compose --env-file .env restart api1 api2 api-gw1 api-gw2 worker lb
+2
View File
@@ -0,0 +1,2 @@
@echo off
"C:\Program Files\Git\bin\bash.exe" "%~dp0restart.sh"
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python3
"""Negative/positive checks for the lab7 trust matrix.
Covers:
* LB -> API mTLS: no client cert, foreign CA, trusted CA but wrong identity,
correct LB identity;
* external HTTPS: untrusted CA is rejected (make test-security also covers it);
* OpenBao: the application policy reads only kv/messenger and is denied
elsewhere; the AppRole login works with the issued credentials.
Run from the repository root after the stand is up:
python scripts/security_internal.py
"""
import json
import ssl
import sys
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PKI = ROOT / "pki"
CA = str(PKI / "ca.crt")
GATEWAY_URL = "https://localhost:9443/health/live"
OPENBAO = "https://127.0.0.1:8200"
def client_context(cert: str | None, key: str | None = None) -> ssl.SSLContext:
ctx = ssl.create_default_context(cafile=CA)
ctx.check_hostname = True
ctx.verify_mode = ssl.CERT_REQUIRED
if cert:
ctx.load_cert_chain(certfile=str(PKI / cert), keyfile=str(PKI / (key or cert.replace(".crt", ".key"))))
return ctx
def get(url: str, ctx: ssl.SSLContext, token: str | None = None):
headers = {"X-Vault-Token": token} if token else {}
request = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(request, context=ctx, timeout=10) as response:
return response.status, response.read().decode()[:200]
except urllib.error.HTTPError as error:
return error.code, error.read().decode()[:200]
except (ssl.SSLError, urllib.error.URLError) as error:
return None, type(error).__name__ + ": " + str(getattr(error, "reason", error))
def attempt(label: str, cert: str | None, expected, key: str | None = None):
ctx = client_context(cert, key)
status, body = get(GATEWAY_URL, ctx)
ok = (status in expected) if isinstance(expected, (list, tuple, set)) else (status == expected)
print(f"{'PASS' if ok else 'FAIL'} {label}: status={status} detail={body[:80]}")
return ok
def openbao_checks() -> bool:
role_id = (ROOT / "secrets" / "openbao_role_id.txt").read_text().strip()
secret_id = (ROOT / "secrets" / "openbao_secret_id.txt").read_text().strip()
ctx = client_context(None)
login = urllib.request.Request(
OPENBAO + "/v1/auth/approle/login",
data=json.dumps({"role_id": role_id, "secret_id": secret_id}).encode(),
headers={"Content-Type": "application/json"}, method="POST")
with urllib.request.urlopen(login, context=ctx, timeout=10) as response:
token = json.loads(response.read())["auth"]["client_token"]
own_status, _ = get(OPENBAO + "/v1/kv/data/messenger", ctx, token)
other_status, _ = get(OPENBAO + "/v1/kv/data/other", ctx, token)
ok = own_status == 200 and other_status == 403
print(f"{'PASS' if ok else 'FAIL'} openbao policy: own={own_status} forbidden={other_status}")
return ok
def main() -> None:
if not (PKI / "ca.crt").exists():
raise SystemExit("PKI is missing: run scripts/pki.sh first")
results = [
attempt("no client certificate", None, [400, 495, 496, 497, None]),
attempt("foreign CA certificate", "foreign-client.crt", [400, 403, 495, 496, 497, None]),
attempt("trusted CA, wrong identity", "other.crt", [403]),
attempt("trusted CA, LB identity", "lb.crt", [200]),
]
results.append(openbao_checks())
if not all(results):
raise SystemExit("security checks failed")
print("PASS: trust matrix negatives and positives verified")
if __name__ == "__main__":
main()
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env python3
"""Check the shared login rate limit (429 + Retry-After) over HTTPS."""
import base64
import json
import ssl
import sys
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
CA = str(ROOT / "pki" / "ca.crt")
BASE = "https://localhost:8443"
def call(method, path, body=None, headers=None):
ctx = ssl.create_default_context(cafile=CA)
data = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(BASE + path, data=data, method=method,
headers={"Content-Type": "application/json", **(headers or {})})
try:
with urllib.request.urlopen(request, context=ctx, timeout=10) as response:
return response.status, dict(response.headers), response.read()
except urllib.error.HTTPError as error:
return error.code, dict(error.headers), error.read()
def main():
username = "rl_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, _, _ = call("POST", "/api/v1/users",
{"username": username, "display_name": "Rate limit", "password": password})
assert status == 201, status
bad = base64.b64encode((username + ":wrong-password-123").encode()).decode()
results = []
for _ in range(6):
status, headers, body = call("POST", "/api/v1/auth/sessions",
headers={"Authorization": "Basic " + bad})
retry_after = headers.get("Retry-After") or headers.get("retry-after")
results.append((status, retry_after))
print("attempts:", results)
limited = results[-1][0] == 429 and results[-1][1] is not None
earlier = all(code == 401 for code, _ in results[:5])
if not (limited and earlier):
raise SystemExit("login rate limit did not behave as expected")
print("PASS: login rate limit returns 429 with Retry-After after repeated failures")
if __name__ == "__main__":
main()
Executable → Regular
View File
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?}"
case "$TARGET_INSTANCE_ID" in
api-1) SVC=api1 ;;
api-2) SVC=api2 ;;
*) echo "unknown instance alias: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
esac
docker compose --env-file .env stop "$SVC"
+2
View File
@@ -0,0 +1,2 @@
@echo off
"C:\Program Files\Git\bin\bash.exe" "%~dp0stop-one.sh"
View File
+47
View File
@@ -0,0 +1,47 @@
import hashlib
import secrets
from datetime import datetime, timezone, timedelta
from fastapi import Depends, Request
from sqlalchemy.orm import Session
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from . import config
from .db import get_db
from .models import User, Session as DbSession
from .errors import ApiError
_ph = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, hash_len=32, salt_len=16)
_BEARER = 'Bearer realm="messenger"'
def hash_password(p: str) -> str:
return _ph.hash(p)
def verify_password(h: str, p: str) -> bool:
try:
return _ph.verify(h, p)
except VerifyMismatchError:
return False
def new_session(db: Session, user: User) -> tuple[str, datetime]:
token = secrets.token_urlsafe(48)
token_hash = hashlib.sha256(token.encode()).hexdigest()
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
db.add(DbSession(token_hash=token_hash, user_id=user.id, expires_at=expires))
db.commit()
return token, expires
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("bearer "):
raise ApiError(401, "unauthorized", "missing bearer token", challenge=_BEARER)
token = auth[7:].strip()
token_hash = hashlib.sha256(token.encode()).hexdigest()
s = db.get(DbSession, token_hash)
if not s:
raise ApiError(401, "unauthorized", "invalid session", challenge=_BEARER)
if s.expires_at <= datetime.now(timezone.utc):
raise ApiError(401, "unauthorized", "expired session", challenge=_BEARER)
u = db.get(User, s.user_id)
if not u:
raise ApiError(401, "unauthorized", "user gone", challenge=_BEARER)
return u
+60
View File
@@ -0,0 +1,60 @@
import os
from pathlib import Path
def _read_secret(env_name: str, file_env: str, default=None):
path = os.environ.get(file_env)
if path:
return Path(path).read_text().strip()
return os.environ.get(env_name, default)
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
APP_ENV = os.environ.get("APP_ENV", "lab")
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
# --- Security (lab7) ---
# When OPENBAO_ADDR is set, runtime credentials come from OpenBao via AppRole.
# The file-based path is kept only for local unit runs without the full stand.
OPENBAO_ADDR = os.environ.get("OPENBAO_ADDR")
OPENBAO_CA = os.environ.get("OPENBAO_CA", "/pki/ca.crt")
PKI_DIR = os.environ.get("PKI_DIR", "/pki")
if OPENBAO_ADDR:
from .secrets import fetch_secrets
_kv, _bao_token, _bao_lease = fetch_secrets()
DATABASE_URL = _kv["database_url"]
S3_ACCESS_KEY = _kv["s3_access_key_id"]
S3_SECRET_KEY = _kv["s3_secret_access_key"]
BROKER_URL = _kv["broker_url"]
else:
DATABASE_URL = _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
S3_ACCESS_KEY = _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
S3_SECRET_KEY = _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
BROKER_URL = _read_secret("BROKER_URL", "BROKER_URL_FILE", "amqp://messenger:messenger@broker:5672/")
_bao_token = None
_bao_lease = 0
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "https://s3:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_CA = os.environ.get("S3_CA", f"{PKI_DIR}/ca.crt")
BROKER_CA = os.environ.get("BROKER_CA", f"{PKI_DIR}/ca.crt")
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
INSTANCE_ID = os.environ.get("INSTANCE_ID", "api-unknown")
# --- lab5: durable broker + worker ---
QUEUE_NAME = os.environ.get("JOB_QUEUE", "attachment_jobs")
STAGING_PREFIX = os.environ.get("STAGING_PREFIX", "staging")
THUMBNAIL_SIZE = int(os.environ.get("THUMBNAIL_SIZE", "256"))
THUMBNAIL_QUALITY = int(os.environ.get("THUMBNAIL_QUALITY", "85"))
MAX_JOB_ATTEMPTS = int(os.environ.get("MAX_JOB_ATTEMPTS", "5"))
OUTBOX_INTERVAL_SECONDS = float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "2"))
OUTBOX_GRACE_SECONDS = float(os.environ.get("OUTBOX_GRACE_SECONDS", "0"))
# --- lab7: login rate limit ---
LOGIN_RATE_LIMIT = int(os.environ.get("LOGIN_RATE_LIMIT", "5"))
LOGIN_RATE_WINDOW_SECONDS = int(os.environ.get("LOGIN_RATE_WINDOW_SECONDS", "60"))
+16
View File
@@ -0,0 +1,16 @@
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, DeclarativeBase
from . import config
engine = create_engine(config.DATABASE_URL, pool_pre_ping=True, pool_size=5, max_overflow=10)
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
class Base(DeclarativeBase):
pass
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
+33
View File
@@ -0,0 +1,33 @@
from fastapi import Request
from fastapi.responses import JSONResponse
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, challenge: str | None = None,
retry_after: int | None = None):
self.status = status
self.code = code
self.message = message
self.challenge = challenge
self.retry_after = retry_after
def _rid(request: Request) -> str:
return getattr(request.state, "request_id", None) or "unknown"
async def api_error_handler(request: Request, exc: ApiError):
headers = {"X-Request-Id": _rid(request)}
if exc.challenge:
headers["WWW-Authenticate"] = exc.challenge
if exc.retry_after is not None:
headers["Retry-After"] = str(exc.retry_after)
return JSONResponse(
status_code=exc.status,
content={"error": {"code": exc.code, "message": exc.message, "request_id": _rid(request)}},
headers=headers,
)
async def unhandled_handler(request: Request, exc: Exception):
return JSONResponse(
status_code=500,
content={"error": {"code": "internal_error", "message": "internal error", "request_id": _rid(request)}},
headers={"X-Request-Id": _rid(request)},
)
+26
View File
@@ -0,0 +1,26 @@
import io
from PIL import Image, UnidentifiedImageError
from . import config
from .errors import ApiError
def validate_image(data: bytes) -> tuple[str, int, int, int]:
"""Возвращает (content_type, width, height, size)."""
if not data:
raise ApiError(400, "invalid_request", "empty body")
if len(data) > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "file too large")
head = data[:8]
if head.startswith(b"\x89PNG\r\n\x1a\n"):
ctype = "image/png"
elif head.startswith(b"\xff\xd8\xff"):
ctype = "image/jpeg"
else:
raise ApiError(415, "unsupported_media_type", "only PNG/JPEG")
try:
img = Image.open(io.BytesIO(data))
w, h = img.size
except UnidentifiedImageError:
raise ApiError(415, "unsupported_media_type", "invalid image")
if w * h > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "too many pixels")
return ctype, w, h, len(data)
+148
View File
@@ -0,0 +1,148 @@
"""Broker access: durable job publication and the transactional-outbox pump.
The broker message carries only identifiers and metadata (never bytes or
credentials). The job row in PostgreSQL is the source of truth; `published_at`
turns the row into a small transactional outbox so a crash between COMMIT and
publish is repaired by the background pump.
"""
import json
import logging
import ssl
import threading
import time
import uuid
from datetime import datetime, timezone, timedelta
from urllib.parse import urlparse, unquote
import pika
from sqlalchemy import select
from . import config
from .db import SessionLocal
from .models import AttachmentJob
log = logging.getLogger("jobs")
TYPE_V1 = "attachment.process.v1"
def pika_params() -> pika.ConnectionParameters:
"""TLS parameters for the broker (amqps) with server verification."""
parsed = urlparse(config.BROKER_URL)
if parsed.scheme == "amqps":
context = ssl.create_default_context(cafile=config.BROKER_CA)
context.check_hostname = True
context.verify_mode = ssl.CERT_REQUIRED
ssl_options = pika.SSLOptions(context, server_hostname=parsed.hostname)
else:
ssl_options = None
params = pika.ConnectionParameters(
host=parsed.hostname or "broker",
port=parsed.port or (5671 if parsed.scheme == "amqps" else 5672),
credentials=pika.PlainCredentials(unquote(parsed.username or "guest"),
unquote(parsed.password or "guest")),
ssl_options=ssl_options,
heartbeat=30,
blocked_connection_timeout=30,
)
return params
def build_payload(job: AttachmentJob, attempt: int, carrier: dict | None = None) -> dict:
from opentelemetry.propagate import inject
if carrier is None:
carrier = {}
inject(carrier) # W3C traceparent when a request span is active
payload = {
"schema_version": 1,
"job_id": str(job.id),
"type": TYPE_V1,
"attachment_id": str(job.attachment_id),
"pipeline_version": job.pipeline_version,
"attempt": attempt,
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
}
if carrier.get("traceparent"):
payload["traceparent"] = carrier["traceparent"]
if carrier.get("tracestate"):
payload["tracestate"] = carrier["tracestate"]
return payload
def publish(payload: dict) -> None:
connection = pika.BlockingConnection(pika_params())
try:
channel = connection.channel()
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
channel.confirm_delivery()
channel.basic_publish(
exchange="",
routing_key=config.QUEUE_NAME,
body=json.dumps(payload).encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2, # persistent
content_type="application/json",
message_id=payload["job_id"],
),
mandatory=True,
)
finally:
connection.close()
def publish_job_row(job_id: uuid.UUID, attempt: int, carrier: dict | None = None) -> None:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if not job or job.state in ("done", "failed"):
return
payload = build_payload(job, attempt, carrier)
publish(payload)
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if job:
job.published_at = datetime.now(timezone.utc)
db.commit()
def pump_once(limit: int = 50) -> int:
"""Publish queued rows whose outbox timestamp is still empty."""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=config.OUTBOX_GRACE_SECONDS)
published = 0
with SessionLocal() as db:
rows = (
db.execute(
select(AttachmentJob)
.where(AttachmentJob.published_at.is_(None))
.where(AttachmentJob.state == "queued")
.where(AttachmentJob.created_at <= cutoff)
.order_by(AttachmentJob.created_at)
.limit(limit)
)
.scalars()
.all()
)
pending = [(job.id, job.attempt + 1) for job in rows]
for job_id, attempt in pending:
try:
publish_job_row(job_id, attempt)
published += 1
except Exception as exc: # broker may still be starting
log.warning("outbox publish failed for %s: %s", job_id, exc)
return published
def start_outbox_thread() -> threading.Thread:
def loop():
while True:
try:
count = pump_once()
if count:
log.info("outbox published %d job(s)", count)
except Exception as exc:
log.warning("outbox pump error: %s", exc)
time.sleep(config.OUTBOX_INTERVAL_SECONDS)
thread = threading.Thread(target=loop, name="outbox", daemon=True)
thread.start()
return thread
+88
View File
@@ -0,0 +1,88 @@
import json
import logging
import time
import uuid
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from . import config, jobs, telemetry
from .db import engine, Base
from .errors import ApiError, api_error_handler, unhandled_handler
from .s3 import ensure_bucket
from .routers import health, users, sessions, chats, messages, attachments
tracer_provider, meter_provider = telemetry.setup_telemetry("messenger-api", config.INSTANCE_ID)
request_counter, request_duration = telemetry.http_instruments()
logger = logging.getLogger("messenger.api")
@asynccontextmanager
async def lifespan(app: FastAPI):
# create_all is not idempotent under parallel startup: two replicas
# may race and one hits UniqueViolation on pg_type. Catch and re-run
# once after a short delay; by then the other replica has finished.
import asyncio
from sqlalchemy.exc import IntegrityError, OperationalError
for attempt in range(5):
try:
Base.metadata.create_all(engine)
break
except (IntegrityError, OperationalError) as exc:
print(f"WARN: create_all attempt {attempt+1} failed: {exc}")
await asyncio.sleep(1.0)
else:
print("WARN: create_all failed after 5 attempts; continuing")
try:
ensure_bucket()
except Exception as exc:
print(f"WARN: ensure_bucket failed: {exc}")
jobs.start_outbox_thread()
logger.info("api.started", extra={"event_fields": json.dumps({"instance_id": config.INSTANCE_ID})})
yield
app = FastAPI(lifespan=lifespan)
app.add_exception_handler(ApiError, api_error_handler)
app.add_exception_handler(Exception, unhandled_handler)
@app.middleware("http")
async def observe_request(request: Request, call_next):
started = time.perf_counter()
response = await call_next(request)
route = request.scope.get("route")
template = getattr(route, "path", request.url.path)
attributes = {
"http.request.method": request.method,
"http.route": template,
"http.response.status_code": response.status_code,
}
request_counter.add(1, attributes)
request_duration.record(time.perf_counter() - started, attributes)
logger.info("http.request", extra={"event_fields": json.dumps({
"http.request.method": request.method,
"http.route": template,
"http.response.status_code": response.status_code,
"request_id": getattr(request.state, "request_id", None)
or request.headers.get("X-Request-Id"),
})})
return response
@app.middleware("http")
async def add_request_id(request: Request, call_next):
rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex
request.state.request_id = rid
response = await call_next(request)
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
for r in (health, users, sessions, chats, messages, attachments):
app.include_router(r.router)
FastAPIInstrumentor.instrument_app(app, excluded_urls="health/live,health/ready")
+149
View File
@@ -0,0 +1,149 @@
import uuid
from datetime import datetime, timezone
from sqlalchemy import String, DateTime, ForeignKey, Integer, UniqueConstraint, Index
from sqlalchemy.dialects.postgresql import UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
from .db import Base
def _uuid() -> uuid.UUID:
return uuid.uuid4()
def _now() -> datetime:
return datetime.now(timezone.utc)
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
display_name: Mapped[str] = mapped_column(String(100))
password_hash: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
class Session(Base):
__tablename__ = "sessions"
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True) # sha256(token) hex
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), index=True
)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
class Chat(Base):
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
title: Mapped[str] = mapped_column(String(100))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
class ChatMember(Base):
__tablename__ = "chat_members"
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
class Message(Base):
__tablename__ = "messages"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
)
sender_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id")
)
text: Mapped[str] = mapped_column(String(4000), default="")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, default=0)
class IdempotencyKey(Base):
__tablename__ = "idempotency_keys"
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
method: Mapped[str] = mapped_column(String(8), primary_key=True)
path: Mapped[str] = mapped_column(String(255), primary_key=True)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
request_hash: Mapped[str] = mapped_column(String(64))
response_json: Mapped[str] = mapped_column(String)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
__table_args__ = (Index("ix_idem_created", "created_at"),)
class Attachment(Base):
__tablename__ = "attachments"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
chat_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
)
owner_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id")
)
filename: Mapped[str] = mapped_column(String(255))
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int] = mapped_column(Integer)
sha256: Mapped[str] = mapped_column(String(64))
s3_key: Mapped[str] = mapped_column(String(255))
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
status: Mapped[str] = mapped_column(String(16), default="ready")
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
class AttachmentJob(Base):
"""Durable intent to process one attachment (transactional outbox row)."""
__tablename__ = "attachment_jobs"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True
)
pipeline_version: Mapped[int] = mapped_column(Integer, default=1)
state: Mapped[str] = mapped_column(String(16), default="queued") # queued|processing|done|failed
attempt: Mapped[int] = mapped_column(Integer, default=0)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
class AttachmentVariant(Base):
__tablename__ = "attachment_variants"
attachment_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True
)
name: Mapped[str] = mapped_column(String(32), primary_key=True)
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int] = mapped_column(Integer)
width: Mapped[int] = mapped_column(Integer)
height: Mapped[int] = mapped_column(Integer)
sha256: Mapped[str] = mapped_column(String(64))
s3_key: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
class LoginAttempt(Base):
__tablename__ = "login_attempts"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
key: Mapped[str] = mapped_column(String(128), index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
View File
+122
View File
@@ -0,0 +1,122 @@
import hashlib
import uuid
from fastapi import APIRouter, Depends, Request, Query
from fastapi.responses import Response
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import Attachment, ChatMember, AttachmentJob, AttachmentVariant
from ..auth import current_user
from ..errors import ApiError
from .. import s3, config, jobs
from ..images import validate_image
from ..schemas import AttachmentOut, VariantOut
router = APIRouter(prefix="/api/v1", tags=["attachments"])
def _bad_filename(name: str) -> bool:
return (not name) or len(name) > 255 or any(c in name for c in ("/", "\\", "\x00")) or any(ord(c) < 32 for c in name)
def _member_chat_or_404(db, chat_id, user_id):
if not db.get(ChatMember, (chat_id, user_id)):
raise ApiError(404, "not_found", "chat not found")
def _attachment_for_member(db, att_id, user_id) -> Attachment:
att = db.get(Attachment, att_id)
if not att:
raise ApiError(404, "not_found", "attachment not found")
_member_chat_or_404(db, att.chat_id, user_id)
return att
def _attachment_out(db, att: Attachment) -> AttachmentOut:
variants = db.query(AttachmentVariant).filter(
AttachmentVariant.attachment_id == att.id
).all()
return AttachmentOut(
id=att.id, chat_id=att.chat_id, owner_id=att.owner_id,
filename=att.filename, content_type=att.content_type,
size_bytes=att.size_bytes, sha256=att.sha256,
status=att.status, error_code=att.error_code,
variants=[VariantOut.model_validate(v) for v in variants],
created_at=att.created_at, updated_at=att.updated_at,
)
@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut)
async def upload(chat_id: uuid.UUID,
filename: str = Query(..., min_length=1, max_length=255),
request: Request = None,
db: Session = Depends(get_db),
user=Depends(current_user)):
_member_chat_or_404(db, chat_id, user.id)
if _bad_filename(filename):
raise ApiError(400, "invalid_request", "invalid filename")
ctype_header = request.headers.get("content-type", "").split(";")[0].strip()
if ctype_header not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "expected image/png or image/jpeg")
data = await request.body()
ctype, w, h, size = validate_image(data)
if ctype != ctype_header:
raise ApiError(415, "unsupported_media_type", "content-type mismatch with file")
digest = hashlib.sha256(data).hexdigest()
att_id = uuid.uuid4()
final_key = f"chats/{chat_id}/attachments/{att_id}"
staging_key = f"{config.STAGING_PREFIX}/{att_id}"
# 1. Durable staging: the original must survive an API restart before 202.
s3.client().put_object(Bucket=config.S3_BUCKET, Key=staging_key, Body=data, ContentType=ctype)
# 2. Metadata + durable job intent in one transaction (transactional outbox).
att = Attachment(id=att_id, chat_id=chat_id, owner_id=user.id, filename=filename,
content_type=ctype, size_bytes=size, sha256=digest, s3_key=final_key,
staging_key=staging_key, status="queued")
job = AttachmentJob(attachment_id=att_id, pipeline_version=1, state="queued", attempt=0)
try:
db.add(att)
db.add(job)
db.commit()
db.refresh(att)
except Exception:
db.rollback()
# Compensating delete for the staging object; the DB has no intent.
try:
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
except Exception:
pass
raise
# 3. Handoff to the broker. If this fails, the row stays unpublished and
# the outbox pump retries it; the client still got a durable 202.
from starlette.concurrency import run_in_threadpool
from opentelemetry.propagate import inject
carrier: dict[str, str] = {}
inject(carrier) # capture the active request span before leaving the event loop
try:
await run_in_threadpool(jobs.publish_job_row, job.id, 1, carrier)
except Exception:
pass
return _attachment_out(db, att)
@router.get("/attachments/{att_id}", response_model=AttachmentOut)
def meta(att_id: uuid.UUID, db=Depends(get_db), user=Depends(current_user)):
return _attachment_out(db, _attachment_for_member(db, att_id, user.id))
@router.get("/attachments/{att_id}/content")
def content(att_id: uuid.UUID, variant: str | None = None,
db=Depends(get_db), user=Depends(current_user)):
att = _attachment_for_member(db, att_id, user.id)
if att.status == "ready":
if variant in (None, "original"):
key, media_type = att.s3_key, att.content_type
else:
row = db.get(AttachmentVariant, (att.id, variant))
if not row:
raise ApiError(404, "not_found", "variant not found")
key, media_type = row.s3_key, row.content_type
elif att.status in ("queued", "processing"):
raise ApiError(409, "conflict", "attachment is not ready")
else:
raise ApiError(409, "conflict", "attachment failed")
obj = s3.client().get_object(Bucket=config.S3_BUCKET, Key=key)
body = obj["Body"].read()
return Response(content=body, media_type=media_type)
+44
View File
@@ -0,0 +1,44 @@
import uuid
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import User, Chat, ChatMember
from ..schemas import CreateChatIn, ChatOut, ChatPage
from ..errors import ApiError
from ..auth import current_user
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
def _chat_out(db, chat) -> ChatOut:
members = [m.user_id for m in db.query(ChatMember).filter(ChatMember.chat_id == chat.id).all()]
return ChatOut(id=chat.id, title=chat.title, member_ids=members, created_at=chat.created_at)
@router.post("", status_code=201, response_model=ChatOut)
def create_chat(body: CreateChatIn, db: Session = Depends(get_db), user: User = Depends(current_user)):
if user.id not in body.member_ids:
raise ApiError(400, "invalid_request", "caller must be a member")
found = db.query(User.id).filter(User.id.in_(body.member_ids)).all()
if len(found) != len(set(body.member_ids)):
raise ApiError(400, "invalid_request", "unknown member")
chat = Chat(title=body.title)
db.add(chat); db.flush()
for uid in body.member_ids:
db.add(ChatMember(chat_id=chat.id, user_id=uid))
db.commit(); db.refresh(chat)
return _chat_out(db, chat)
@router.get("", response_model=ChatPage)
def list_chats(db: Session = Depends(get_db), user: User = Depends(current_user)):
rows = db.query(Chat).join(ChatMember, ChatMember.chat_id == Chat.id)\
.filter(ChatMember.user_id == user.id)\
.order_by(Chat.created_at, Chat.id).limit(100).all()
return ChatPage(items=[_chat_out(db, c) for c in rows], next_cursor=None)
@router.get("/{chat_id}", response_model=ChatOut)
def get_chat(chat_id: uuid.UUID, db: Session = Depends(get_db), user: User = Depends(current_user)):
if not db.get(ChatMember, (chat_id, user.id)):
raise ApiError(404, "not_found", "chat not found")
chat = db.get(Chat, chat_id)
if not chat:
raise ApiError(404, "not_found", "chat not found")
return _chat_out(db, chat)
+19
View File
@@ -0,0 +1,19 @@
from fastapi import APIRouter, Depends
from sqlalchemy import text
from sqlalchemy.orm import Session
from ..db import get_db
from ..errors import ApiError
router = APIRouter(prefix="/health", tags=["health"])
@router.get("/live")
def live():
return {"status": "ok"}
@router.get("/ready")
def ready(db: Session = Depends(get_db)):
try:
db.execute(text("SELECT 1"))
except Exception:
raise ApiError(503, "unavailable", "database unavailable")
return {"status": "ok"}
+147
View File
@@ -0,0 +1,147 @@
import uuid
import json
import hashlib
import base64
from datetime import datetime
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from ..db import get_db
from ..models import (
User,
ChatMember,
Message,
MessageAttachment,
IdempotencyKey,
Attachment,
)
from ..schemas import CreateMessageIn, MessageOut, MessagePage
from ..errors import ApiError
from ..auth import current_user
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
def _msg_out(db: Session, m: Message) -> MessageOut:
atts = [
a.attachment_id
for a in db.query(MessageAttachment)
.filter(MessageAttachment.message_id == m.id)
.order_by(MessageAttachment.position)
.all()
]
return MessageOut(
id=m.id,
chat_id=m.chat_id,
sender_id=m.sender_id,
text=m.text,
created_at=m.created_at,
attachment_ids=atts,
)
def _require_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
if not db.get(ChatMember, (chat_id, user_id)):
raise ApiError(404, "not_found", "chat not found")
@router.post("", status_code=201, response_model=MessageOut)
def send(
chat_id: uuid.UUID,
body: CreateMessageIn,
request: Request,
db: Session = Depends(get_db),
user: User = Depends(current_user),
):
_require_member(db, chat_id, user.id)
key = request.headers.get("idempotency-key", "")
if not key or len(key) > 128:
raise ApiError(400, "invalid_request", "Idempotency-Key required")
if not body.text.strip() and not body.attachment_ids:
raise ApiError(400, "invalid_request", "empty message")
for aid in body.attachment_ids:
a = db.get(Attachment, aid)
if not a or a.chat_id != chat_id:
raise ApiError(404, "not_found", "attachment not found")
path = f"/api/v1/chats/{chat_id}/messages"
body_hash = hashlib.sha256(
json.dumps(body.model_dump(), sort_keys=True, default=str).encode()
).hexdigest()
existing = db.get(IdempotencyKey, (user.id, "POST", path, key))
if existing:
if existing.request_hash != body_hash:
raise ApiError(409, "conflict", "idempotency key reused with different body")
return json.loads(existing.response_json)
m = Message(chat_id=chat_id, sender_id=user.id, text=body.text)
db.add(m)
db.flush() # чтобы появился m.id
for i, aid in enumerate(body.attachment_ids):
db.add(MessageAttachment(message_id=m.id, attachment_id=aid, position=i))
db.flush() # чтобы _msg_out увидел MessageAttachment
out = _msg_out(db, m)
db.add(
IdempotencyKey(
user_id=user.id,
method="POST",
path=path,
key=key,
request_hash=body_hash,
response_json=json.dumps(out.model_dump(), default=str),
)
)
db.commit()
return out
@router.get("", response_model=MessagePage)
def list_messages(
chat_id: uuid.UUID,
limit: int = 50,
cursor: str | None = None,
db: Session = Depends(get_db),
user: User = Depends(current_user),
):
_require_member(db, chat_id, user.id)
if limit < 1 or limit > 100:
raise ApiError(400, "invalid_request", "bad limit")
q = (
db.query(Message)
.filter(Message.chat_id == chat_id)
.order_by(Message.created_at, Message.id)
)
if cursor:
try:
c = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
ts = datetime.fromisoformat(c["t"])
mid = uuid.UUID(c["i"])
q = q.filter(
(Message.created_at > ts)
| ((Message.created_at == ts) & (Message.id > mid))
)
except Exception:
raise ApiError(400, "invalid_request", "bad cursor")
rows = q.limit(limit + 1).all()
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = None
if has_more:
last = rows[-1]
payload = json.dumps({"t": last.created_at.isoformat(), "i": str(last.id)}).encode()
next_cursor = base64.urlsafe_b64encode(payload).decode()
return MessagePage(items=[_msg_out(db, m) for m in rows], next_cursor=next_cursor)
+72
View File
@@ -0,0 +1,72 @@
import base64
import hashlib
from datetime import datetime, timedelta, timezone
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from .. import config
from ..db import get_db
from ..models import User, Session as DbSession, LoginAttempt
from ..schemas import SessionOut
from ..errors import ApiError
from ..auth import verify_password, new_session, current_user
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
_BASIC = 'Basic realm="messenger", charset="UTF-8"'
_BEARER = 'Bearer realm="messenger"'
def _client_ip(request: Request) -> str:
forwarded = request.headers.get("x-forwarded-for", "")
if forwarded:
return forwarded.split(",")[0].strip()[:64]
return (request.client.host if request.client else "unknown")[:64]
def _enforce_login_rate(db: Session, key: str) -> None:
now = datetime.now(timezone.utc)
window_start = now - timedelta(seconds=config.LOGIN_RATE_WINDOW_SECONDS)
attempts = (
db.query(LoginAttempt)
.filter(LoginAttempt.key == key)
.filter(LoginAttempt.created_at > window_start)
.count()
)
if attempts >= config.LOGIN_RATE_LIMIT:
raise ApiError(429, "rate_limited", "too many login attempts", retry_after=config.LOGIN_RATE_WINDOW_SECONDS)
def _record_failure(db: Session, key: str) -> None:
db.add(LoginAttempt(key=key))
db.commit()
@router.post("", status_code=201, response_model=SessionOut)
def login(request: Request, db: Session = Depends(get_db)):
auth = request.headers.get("authorization", "")
if not auth.lower().startswith("basic "):
raise ApiError(401, "unauthorized", "basic required", challenge=_BASIC)
try:
raw = base64.b64decode(auth[6:].strip()).decode("utf-8")
username, password = raw.split(":", 1)
except Exception:
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
key = f"{username}:{_client_ip(request)}"
_enforce_login_rate(db, key)
user = db.query(User).filter(User.username == username).first()
if not user or not verify_password(user.password_hash, password):
_record_failure(db, key)
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
token, expires = new_session(db, user)
return {"token": token, "token_type": "Bearer", "expires_at": expires, "user": user}
@router.delete("/current", status_code=204)
def logout(request: Request, db: Session = Depends(get_db), user: User = Depends(current_user)):
auth = request.headers.get("authorization", "")
token = auth[7:].strip()
h = hashlib.sha256(token.encode()).hexdigest()
s = db.get(DbSession, h)
if s:
db.delete(s)
db.commit()
return None
+27
View File
@@ -0,0 +1,27 @@
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from sqlalchemy.exc import IntegrityError
from ..db import get_db
from ..models import User
from ..schemas import CreateUserIn, UserOut
from ..errors import ApiError
from ..auth import hash_password, current_user
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", status_code=201, response_model=UserOut)
def register(body: CreateUserIn, db: Session = Depends(get_db)):
u = User(username=body.username, display_name=body.display_name,
password_hash=hash_password(body.password))
db.add(u)
try:
db.commit()
except IntegrityError:
db.rollback()
raise ApiError(409, "conflict", "username taken")
db.refresh(u)
return u
@router.get("/me", response_model=UserOut)
def me(user: User = Depends(current_user)):
return user
+37
View File
@@ -0,0 +1,37 @@
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError
from . import config
_client = None
def client():
global _client
if _client is None:
_client = boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY,
aws_secret_access_key=config.S3_SECRET_KEY,
verify=config.S3_CA,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
retries={"max_attempts": 2, "mode": "standard"},
connect_timeout=3,
read_timeout=10,
),
)
return _client
def ensure_bucket():
c = client()
try:
c.head_bucket(Bucket=config.S3_BUCKET)
except ClientError as e:
code = e.response.get("Error", {}).get("Code", "")
if code in ("404", "NoSuchBucket", "NotFound"):
c.create_bucket(Bucket=config.S3_BUCKET)
else:
raise
+76
View File
@@ -0,0 +1,76 @@
import uuid
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field, ConfigDict
class UserOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: uuid.UUID
username: str
display_name: str
created_at: datetime
class CreateUserIn(BaseModel):
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
class SessionOut(BaseModel):
token: str
token_type: str = "Bearer"
expires_at: datetime
user: UserOut
class ChatOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: uuid.UUID
title: str
member_ids: list[uuid.UUID]
created_at: datetime
class CreateChatIn(BaseModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
class MessageOut(BaseModel):
id: uuid.UUID
chat_id: uuid.UUID
sender_id: uuid.UUID
text: str
created_at: datetime
attachment_ids: list[uuid.UUID]
class CreateMessageIn(BaseModel):
text: str = Field(default="", max_length=4000)
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
class MessagePage(BaseModel):
items: list[MessageOut]
next_cursor: Optional[str] = None
class ChatPage(BaseModel):
items: list[ChatOut]
next_cursor: Optional[str] = None
class VariantOut(BaseModel):
model_config = ConfigDict(from_attributes=True)
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class AttachmentOut(BaseModel):
id: uuid.UUID
chat_id: uuid.UUID
owner_id: uuid.UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str = "ready"
variants: list[VariantOut] = []
error_code: Optional[str] = None
created_at: datetime
updated_at: datetime
+51
View File
@@ -0,0 +1,51 @@
"""Service-identity secret loading from OpenBao (AppRole -> KV v2).
The application never uses a root token or unseal keys. It logs in with a
short-lived SecretID issued by the operator bootstrap, receives a TTL-bound
token and reads only its own KV path.
"""
import json
import os
import ssl
import urllib.request
from pathlib import Path
def _read(path: str) -> str:
return Path(path).read_text().strip()
def _context(ca_file: str) -> ssl.SSLContext:
ctx = ssl.create_default_context(cafile=ca_file)
ctx.check_hostname = True
ctx.verify_mode = ssl.CERT_REQUIRED
return ctx
def _call(method: str, url: str, ctx: ssl.SSLContext, token: str | None = None,
body: dict | None = None, timeout: int = 10):
headers = {"Content-Type": "application/json"}
if token:
headers["X-Vault-Token"] = token
data = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(url, method=method, data=data, headers=headers)
with urllib.request.urlopen(request, context=ctx, timeout=timeout) as response:
return json.loads(response.read().decode())
def fetch_secrets() -> tuple[dict, str, int]:
"""Return (kv data, token, lease_seconds) for the messenger path."""
addr = os.environ.get("OPENBAO_ADDR", "https://openbao:8200")
ca = os.environ.get("OPENBAO_CA", "/pki/ca.crt")
role_id_file = os.environ.get("OPENBAO_ROLE_ID_FILE", "/run/secrets/openbao_role_id")
secret_id_file = os.environ.get("OPENBAO_SECRET_ID_FILE", "/run/secrets/openbao_secret_id")
ctx = _context(ca)
login = _call("POST", addr + "/v1/auth/approle/login", ctx, body={
"role_id": _read(role_id_file),
"secret_id": _read(secret_id_file),
})
token = login["auth"]["client_token"]
lease = int(login["auth"].get("lease_duration", 3600))
kv = _call("GET", addr + "/v1/kv/data/messenger", ctx, token=token)
return kv["data"]["data"], token, lease
+136
View File
@@ -0,0 +1,136 @@
"""OpenTelemetry wiring for API and worker (lab6).
Signals go OTLP -> collector. Metrics are exposed by the collector as a
Prometheus endpoint, traces are stored in Tempo, logs in Loki. The JSON logs
kept in stdout also carry trace_id/span_id so a trace can be matched with a
line even without the log backend.
"""
import json
import logging
import os
from datetime import datetime, timezone
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import (
Resource,
SERVICE_INSTANCE_ID,
SERVICE_NAME,
SERVICE_VERSION,
)
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.propagate import set_global_textmap
class TraceJsonFormatter(logging.Formatter):
"""One JSON object per line, with trace correlation and no bodies."""
def __init__(self, service_name: str):
super().__init__()
self.service_name = service_name
def format(self, record: logging.LogRecord) -> str:
ctx = trace.get_current_span().get_span_context()
payload = {
"timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"severity": record.levelname,
"service.name": self.service_name,
"event": record.getMessage(),
"logger": record.name,
}
if ctx and ctx.is_valid:
payload["trace_id"] = format(ctx.trace_id, "032x")
payload["span_id"] = format(ctx.span_id, "016x")
extra = getattr(record, "event_fields", None)
if isinstance(extra, dict):
payload.update({k: v for k, v in extra.items() if not isinstance(v, (bytes, bytearray))})
elif isinstance(extra, str):
try:
payload.update({k: v for k, v in json.loads(extra).items()})
except ValueError:
pass
if record.exc_info and record.exc_info[0]:
payload["error.type"] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False)
def _otlp_credentials():
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
if not endpoint.startswith("https://"):
return endpoint, None
import grpc
ca_path = os.environ.get("OTEL_CA", "/pki/ca.crt")
with open(ca_path, "rb") as handle:
root = handle.read()
return endpoint, grpc.ssl_channel_credentials(root_certificates=root)
def setup_telemetry(service_name: str, instance_id: str):
endpoint, credentials = _otlp_credentials()
insecure = credentials is None
resource = Resource.create({
SERVICE_NAME: service_name,
SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "6.0.0"),
SERVICE_INSTANCE_ID: instance_id,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
})
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, credentials=credentials, insecure=insecure))
)
trace.set_tracer_provider(tracer_provider)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint, credentials=credentials, insecure=insecure),
export_interval_millis=int(os.environ.get("OTEL_METRIC_INTERVAL_MS", "10000")),
)],
)
metrics.set_meter_provider(meter_provider)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint, credentials=credentials, insecure=insecure))
)
root = logging.getLogger()
root.setLevel(logging.INFO)
for handler in list(root.handlers):
root.removeHandler(handler)
stream = logging.StreamHandler()
stream.setFormatter(TraceJsonFormatter(service_name))
root.addHandler(stream)
root.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logger_provider))
set_global_textmap(TraceContextTextMapPropagator())
return tracer_provider, meter_provider
def http_instruments():
meter = metrics.get_meter("messenger.http")
return (
meter.create_counter("http.server.requests", unit="1",
description="HTTP requests by method, route template and status"),
meter.create_histogram("http.server.request.duration", unit="s",
description="HTTP server request duration"),
)
def job_instruments():
meter = metrics.get_meter("messenger.worker")
return (
meter.create_counter("attachment.jobs", unit="1",
description="Attachment jobs by outcome"),
meter.create_histogram("attachment.job.duration", unit="s",
description="Attachment job duration by outcome"),
)
+38
View File
@@ -0,0 +1,38 @@
"""Deterministic image variants (lab5: one thumbnail per the contract)."""
import hashlib
import io
from PIL import Image, ImageOps
from . import config
def _flatten_on_white(img: Image.Image) -> Image.Image:
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
return Image.alpha_composite(background, rgba).convert("RGB")
return img.convert("RGB")
def make_thumbnail(data: bytes) -> tuple[bytes, int, int, str]:
"""Fit the image into 256x256 (no upscaling), JPEG on white.
Returns (jpeg_bytes, width, height, sha256_hex).
"""
img = Image.open(io.BytesIO(data))
img = ImageOps.exif_transpose(img)
img = _flatten_on_white(img)
limit = config.THUMBNAIL_SIZE
width, height = img.size
scale = min(limit / width, limit / height, 1.0)
new_width = max(1, int(width * scale))
new_height = max(1, int(height * scale))
if (new_width, new_height) != (width, height):
img = img.resize((new_width, new_height), Image.LANCZOS)
out = io.BytesIO()
img.save(out, "JPEG", quality=config.THUMBNAIL_QUALITY, optimize=True)
payload = out.getvalue()
return payload, new_width, new_height, hashlib.sha256(payload).hexdigest()
+226
View File
@@ -0,0 +1,226 @@
"""Durable attachment worker (lab5/lab6).
Consumes ``attachment.process.v1`` messages from a persistent queue, performs
the heavy image work outside the HTTP path, records the result atomically in
PostgreSQL, and emits OTLP traces/metrics/logs. Delivery is at-least-once; the
effect is idempotent by ``(attachment_id, pipeline_version, variant)`` and
every state transition is committed before the broker ack.
"""
import json
import logging
import signal
import time
import uuid
import pika
from opentelemetry import trace
from opentelemetry.propagate import extract
from opentelemetry.trace import SpanKind, Status, StatusCode
from sqlalchemy import select
from . import config, s3, telemetry
from .db import SessionLocal
from .errors import ApiError
from .images import validate_image
from .models import Attachment, AttachmentJob, AttachmentVariant
from .thumbnails import make_thumbnail
_, _meter_provider = telemetry.setup_telemetry("messenger-worker", "worker")
job_counter, job_duration = telemetry.job_instruments()
tracer = trace.get_tracer("messenger.worker")
log = logging.getLogger("messenger.worker")
JOB_TYPE = "attachment.process.v1"
_stop = False
def _handle_signal(signum, frame):
global _stop
_stop = True
log.info("worker.stop_requested")
def _failure_code(exc: Exception) -> tuple[str, bool]:
"""Return (safe error_code, permanent)."""
if isinstance(exc, ApiError):
if exc.status in (400, 413, 415):
return "invalid_image", True
return "processing_failed", False
if isinstance(exc, (OSError, ValueError)):
return "invalid_image", True
return "processing_failed", False
def _set_state(job_id: uuid.UUID, attachment_id: uuid.UUID, job_state: str,
att_state: str | None = None, attempt: int | None = None,
error: str | None = None) -> None:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
if not job:
return
job.state = job_state
if attempt is not None:
job.attempt = max(job.attempt or 0, attempt)
if error is not None:
job.last_error = error[:255]
att = db.get(Attachment, attachment_id)
if att and att_state:
att.status = att_state
att.error_code = error if att_state == "failed" else None
db.commit()
def process(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> str:
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
att = db.get(Attachment, attachment_id)
if not job or not att:
log.warning("job.dropped", extra={"event_fields": json.dumps({"reason": "row_gone"})})
return "dropped"
if att.status == "ready":
job.state = "done"
db.commit()
log.info("job.duplicate", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
return "duplicate"
if att.status == "failed":
job.state = "failed"
db.commit()
log.info("job.duplicate_failed", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
return "skipped"
job.state = "processing"
job.attempt = max(job.attempt or 0, attempt)
att.status = "processing"
db.commit()
pipeline_version = job.pipeline_version
staging_key = att.staging_key or f"{config.STAGING_PREFIX}/{attachment_id}"
final_key = att.s3_key
try:
with SessionLocal() as db:
att = db.get(Attachment, attachment_id)
declared_type = att.content_type if att else None
data = s3.client().get_object(Bucket=config.S3_BUCKET, Key=staging_key)["Body"].read()
ctype, _w, _h, _size = validate_image(data)
s3.client().put_object(Bucket=config.S3_BUCKET, Key=final_key, Body=data,
ContentType=declared_type or ctype)
thumb, tw, th, tsha = make_thumbnail(data)
thumb_key = f"variants/{attachment_id}/thumbnail"
s3.client().put_object(Bucket=config.S3_BUCKET, Key=thumb_key, Body=thumb, ContentType="image/jpeg")
with SessionLocal() as db:
att = db.get(Attachment, attachment_id)
job = db.get(AttachmentJob, job_id)
existing = db.get(AttachmentVariant, (attachment_id, "thumbnail"))
if existing is None:
db.add(AttachmentVariant(
attachment_id=attachment_id, name="thumbnail",
content_type="image/jpeg", size_bytes=len(thumb),
width=tw, height=th, sha256=tsha, s3_key=thumb_key,
))
elif existing.s3_key != thumb_key:
log.warning("variant.kept", extra={"event_fields": json.dumps({"reason": "newer_pipeline"})})
att.status = "ready"
att.error_code = None
job.state = "done"
job.last_error = None
job.attempt = max(job.attempt or 0, attempt)
db.commit()
log.info("job.ready", extra={"event_fields": json.dumps({
"attachment_id": str(attachment_id),
"pipeline_version": pipeline_version,
"attempt": attempt,
})})
try:
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
except Exception:
pass
return "ready"
except Exception as exc:
code, permanent = _failure_code(exc)
log.exception("job.failed", extra={"event_fields": json.dumps({
"attachment_id": str(attachment_id),
"attempt": attempt,
"error_code": code,
})})
if permanent or attempt >= config.MAX_JOB_ATTEMPTS:
_set_state(job_id, attachment_id, "failed", att_state="failed",
attempt=attempt, error=code)
return "failed"
_set_state(job_id, attachment_id, "queued", att_state="queued",
attempt=attempt, error=code)
raise
def handle(channel, method, properties, body):
outcome = "retry"
try:
message = json.loads(body)
job_id = uuid.UUID(message["job_id"])
attachment_id = uuid.UUID(message["attachment_id"])
attempt = int(message.get("attempt", 1))
except Exception as exc:
log.warning("job.poison", extra={"event_fields": json.dumps({"reason": type(exc).__name__})})
channel.basic_ack(delivery_tag=method.delivery_tag)
return
context = extract(message)
started = time.perf_counter()
with tracer.start_as_current_span("attachment.process", context=context,
kind=SpanKind.CONSUMER) as span:
span.set_attribute("job.type", JOB_TYPE)
span.set_attribute("attachment.id", str(attachment_id))
span.set_attribute("job.attempt", attempt)
try:
outcome = process(job_id, attachment_id, attempt)
span.set_attribute("job.outcome", outcome)
if outcome == "failed":
span.set_status(Status(StatusCode.ERROR))
channel.basic_ack(delivery_tag=method.delivery_tag)
except Exception as exc:
outcome = "retry"
span.set_attribute("job.outcome", outcome)
span.set_status(Status(StatusCode.ERROR))
span.record_exception(exc)
from .jobs import build_payload, publish
with SessionLocal() as db:
job = db.get(AttachmentJob, job_id)
payload = build_payload(job, job.attempt + 1) if job and job.state == "queued" else None
if payload:
try:
publish(payload)
except Exception:
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
outcome = "requeued"
return
channel.basic_ack(delivery_tag=method.delivery_tag)
finally:
attributes = {"job.type": JOB_TYPE, "job.outcome": outcome}
job_counter.add(1, attributes)
job_duration.record(time.perf_counter() - started, attributes)
def main():
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
from .jobs import pika_params
while not _stop:
try:
connection = pika.BlockingConnection(pika_params())
channel = connection.channel()
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=config.QUEUE_NAME, on_message_callback=handle, auto_ack=False)
log.info("worker.consuming", extra={"event_fields": json.dumps({"queue": config.QUEUE_NAME})})
while not _stop:
connection.process_data_events(time_limit=1)
connection.close()
except Exception as exc:
log.warning("broker.reconnect", extra={"event_fields": json.dumps({"error": type(exc).__name__})})
time.sleep(3)
log.info("worker.stopped")
if __name__ == "__main__":
main()
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, Failure, LAB, arguments, check, run
def history(c, actor, chat):
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
result = []
seen = set()
for _ in range(1000):
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
result.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return result
check(cursor not in seen,'cursor cycle')
seen.add(cursor)
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
raise Failure('history pagination exceeded safety bound')
def main():
def configure(p):
p.add_argument('--mode', choices=['persistence','failover'], required=True)
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
check(LAB>=2,'persistence requires lab2+')
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
check('.example.' not in script.name,'copy and implement the example hook first')
c = Client(args.base_url,args.ca_file,timeout=10)
alice,bob,eve,chat = c.bootstrap()
sent = c.message(alice,chat,'Сохранить при отказе')
original = sent['json']
target = sent['headers'].get('X-Instance-Id','')
attachment = content = None
if LAB>=4:
attachment,content = c.attachment(alice,chat)
if LAB>=5:
attachment = c.wait_ready(alice,attachment)
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
observed = set()
if args.mode=='failover':
for _ in range(40):
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
# No shell interpolation and no Docker commands in this test.
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
started = time.monotonic()
successes,failures,after_instances = [],0,[]
try:
if args.mode=='persistence':
try:
code = proc.wait(timeout=90)
except subprocess.TimeoutExpired:
raise Failure('restart hook exceeded 90 seconds') from None
check(code==0,'restart hook failed (inspect your script locally)')
deadline = time.monotonic()+90
while True:
try:
c.call('GET','/health/ready',schema='Health')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
break
except Failure:
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
time.sleep(0.5)
else:
first_recovery = None
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
while time.monotonic()-started < 15:
code = proc.poll()
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
try:
response = c.message(alice,chat,text,key=key)
msg = response['json']
if code==0:
if first_recovery is None:
first_recovery = time.monotonic()-started
after_instances.append(response['headers']['X-Instance-Id'])
successes.append(msg)
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
except Failure:
failures += 1
# Keep SAME key/body after an uncertain POST result.
time.sleep(0.1)
check(proc.poll()==0,'stop-one hook did not finish successfully')
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
check(len(after_instances)>=5,'too few successful writes after stop')
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
# Existing token, no re-login. Read all confirmations and reject duplicates.
messages = history(c,bob,chat)
ids = [m['id'] for m in messages]
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
for msg in successes:
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
texts = [m['text'] for m in messages]
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
if attachment:
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
check(downloaded['bytes']==content,'attachment lost or changed across failure')
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
finally:
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
if __name__=='__main__':
run(main)