forked from mtusi2026/mtusi2026lab7
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
861e7722b5 |
+48
-22
@@ -1,41 +1,67 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
GRACEFUL_TIMEOUT=8
|
||||
MAX_INFLIGHT_REQUESTS=64
|
||||
MAX_BODY_BYTES=65536
|
||||
MAX_RSS_MB=0
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT=3
|
||||
DB_CONNECT_TIMEOUT=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
DB_OP_TIMEOUT=6
|
||||
REQUEST_TIMEOUT=30
|
||||
DB_READY_TIMEOUT=1.5
|
||||
READY_CHECK_SECONDS=1
|
||||
READY_FAILURES=2
|
||||
SESSION_CACHE_TTL_SECONDS=30
|
||||
SESSION_SWEEP_SECONDS=60
|
||||
CACHE_TIMEOUT=0.2
|
||||
SESSION_REVOKED_TTL_SECONDS=60
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
UPLOAD_CONCURRENCY=2
|
||||
RECONCILE_INTERVAL_SECONDS=300
|
||||
RECONCILE_GRACE_SECONDS=900
|
||||
# --- lab5: broker, jobs, worker
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=20
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
JOB_BACKOFF_MAX_SECONDS=30
|
||||
JOB_SWEEP_SECONDS=5
|
||||
WORKER_CHILD_MEMORY_MB=1024
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing.
|
||||
MAX_PENDING_JOBS=200
|
||||
JPEG_QUALITY=85
|
||||
WEBP_QUALITY=80
|
||||
# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else.
|
||||
FAULT_INJECTION=1
|
||||
# Session store: PostgreSQL is the source of truth, Valkey (TLS + ACL user `api`) caches sessions and holds the
|
||||
# shared login rate limit counters.
|
||||
SESSION_STORE=valkey
|
||||
# --- lab7: TLS and OpenBao. No passwords or tokens here: compose.yaml passes only file PATHS (Docker secrets).
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
# Argon2id (RFC 9106 / OWASP minimum: m=19 MiB, t=2, p=1). Measured on the stand, see REPORT.md.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
# Concurrent password hashes per replica (login DoS budget) and login attempts per window, shared via Valkey.
|
||||
PASSWORD_HASH_CONCURRENCY=4
|
||||
LOGIN_RATE_LIMIT_PER_USER=5
|
||||
LOGIN_RATE_LIMIT_PER_IP=120
|
||||
LOGIN_RATE_WINDOW_SECONDS=60
|
||||
# OpenBao: secrets are re-read on this period (rotation pick-up); startup waits this long for an unsealed OpenBao.
|
||||
BAO_REFRESH_SECONDS=15
|
||||
BAO_STARTUP_TIMEOUT_SECONDS=120
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
name: Application stand (lab7)
|
||||
# Student job: the whole operator bootstrap in a throwaway runner, then the public HTTPS checks and the TLS audit.
|
||||
# The job itself plays the operator: PKI, bootstrap passwords and OpenBao shares are created inside the runner and
|
||||
# destroyed with it. No repository secrets are used; pull_request_target is deliberately not a trigger.
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
stand:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Local PKI and bootstrap credentials (operator step)
|
||||
run: |
|
||||
sh scripts/pki.sh init
|
||||
sh scripts/init-secrets.sh
|
||||
- name: Validate Compose configuration
|
||||
run: docker compose --env-file .env config --quiet
|
||||
- name: OpenBao init, unseal and bootstrap (operator step)
|
||||
run: |
|
||||
docker compose --env-file .env up -d openbao
|
||||
sh scripts/bao.sh init
|
||||
sh scripts/bao.sh bootstrap
|
||||
- name: Build and start the stand, wait for readiness
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
|
||||
- name: Contract structure and public HTTPS checks
|
||||
run: |
|
||||
make check
|
||||
make test CA_FILE=$PWD/secrets/pki/ca.crt
|
||||
make test-security CA_FILE=$PWD/secrets/pki/ca.crt
|
||||
- name: TLS / mTLS / authorization audit of the trust matrix
|
||||
run: >
|
||||
docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro
|
||||
-v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py
|
||||
- name: Container hardening from docker inspect
|
||||
run: python3 student-tests/check_hardening.py
|
||||
- name: Service state and logs on failure (no secrets are logged)
|
||||
if: failure()
|
||||
run: |
|
||||
docker compose ps -a
|
||||
docker compose logs --tail 60 nginx api-1 api-2 worker openbao rabbitmq postgres
|
||||
- name: Remove the stand
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
FROM python:3.12.16-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=120 \
|
||||
PIP_RETRIES=10
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-deps -r requirements.txt
|
||||
|
||||
COPY app/ app/
|
||||
COPY migrations/ migrations/
|
||||
COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
CMD ["python", "-m", "app.serve"]
|
||||
@@ -1,37 +1,210 @@
|
||||
# Отчёт — лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Алёна Курлова, БВТ2301
|
||||
- Целевая оценка; выбранный трек 5: **4** (трек 5 не заявлен)
|
||||
- Commit/PR: после публикации (ветка `solution/bvt2301-kurlova`)
|
||||
- Источники, библиотеки, помощь ИИ: OpenBao 2.6.5 ([AppRole/cert auth](https://openbao.org/docs/auth/cert/), [policies](https://openbao.org/docs/concepts/policies/), [production hardening](https://openbao.org/docs/internals/security/)), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [OWASP File Upload](https://cheatsheetseries.owasp.org/cheatsheets/File_Upload_Cheat_Sheet.html), [Docker security](https://docs.docker.com/engine/security/), RabbitMQ [TLS](https://www.rabbitmq.com/docs/ssl) и [access control](https://www.rabbitmq.com/docs/access-control), PostgreSQL [SSL](https://www.postgresql.org/docs/17/ssl-tcp.html), argon2-cffi, Trivy 0.58.1. Код lab6 перенесён, бизнес-API не менялся. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора. Все результаты ниже получены на локальном стенде и сохранены в `evidence/`.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Desktop (Compose v2.20+), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM 10 CPU / 7.7 GiB.
|
||||
|
||||
**Шаги оператора** (секреты и ключи создаются локально, в Git и образы не попадают; скрипты ничего секретного не печатают):
|
||||
|
||||
```sh
|
||||
sh scripts/pki.sh init # CA (Root + Ingest + Telemetry Clients) и leaf-сертификаты -> secrets/pki/ (0700)
|
||||
sh scripts/init-secrets.sh # .env и bootstrap-пароли хранилищ -> secrets/bootstrap/ (0700)
|
||||
docker compose --env-file .env up -d openbao
|
||||
sh scripts/bao.sh init # 3 доли unseal (порог 2) + root token -> secrets/operator/ (0600); затем unseal
|
||||
sh scripts/bao.sh bootstrap # KV v2 `messenger/`, политики, cert auth (роли api/worker), секреты сервисов
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make check
|
||||
make test CA_FILE=$PWD/secrets/pki/ca.crt
|
||||
make test-security CA_FILE=$PWD/secrets/pki/ca.crt
|
||||
```
|
||||
|
||||
**Bootstrap, повторяемый при каждом `up`:**
|
||||
- `migrate` — SQL-миграции, роли `app_api`/`app_worker` и их пароли из `secrets/bootstrap`; подключение по TLS `verify-full`;
|
||||
- `minio-init` — бакет, политики, пользователи `messenger-api` и `messenger-worker`; по HTTPS;
|
||||
- RabbitMQ при каждом старте импортирует пользователей, права и топологию из `infra/rabbitmq/definitions.json`;
|
||||
- Grafana подхватывает provisioning.
|
||||
|
||||
**Что делает оператор после рестарта OpenBao:** `sh scripts/bao.sh unseal`. Уже работающие сервисы продолжают работать, перезапущенные ждут unseal. Restart hook (`scripts/restart.sh`) делает unseal сам, потому что его запускает оператор.
|
||||
|
||||
**Проверки:**
|
||||
|
||||
```sh
|
||||
docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro \
|
||||
-v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py # вся матрица доверия, 135 проверок
|
||||
python3 student-tests/test_auth_acl.py # ACL, X-User-Id, logout/expiry, лимит входа, размеры, rehash
|
||||
python3 student-tests/test_cert_rotation.py # ротация и отзыв leaf lb-client, просроченный сертификат
|
||||
python3 student-tests/check_hardening.py # docker inspect всех контейнеров
|
||||
python3 student-tests/test_observability.py # трассы/логи/метрики через mTLS, отсутствие утечек
|
||||
python3 student-tests/check_dashboard.py # все запросы dashboard, включая выполнение через Grafana
|
||||
python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5
|
||||
sh student-tests/operator_checks.sh # ОПЕРАТОР: ротация пароля worker, seal/unseal, persistence, failover
|
||||
```
|
||||
|
||||
CI: `.github/workflows/stand.yml` повторяет всю последовательность оператора в одноразовом runner (PKI, OpenBao init/bootstrap, стенд, `make test`, `make test-security`, TLS-аудит, hardening). Локально эти шаги проверены; на runner workflow ещё не выполнялся.
|
||||
|
||||
Для LOGIN DoS-замера: `LOGIN_RATE_LIMIT_PER_IP=1000000 docker compose --env-file .env up -d --no-deps api-1 api-2 && python3 student-tests/login_budget.py`, затем вернуть лимит обычным `up`.
|
||||
|
||||
**Остановка и удаление.** Безопасная остановка с сохранением данных: `docker compose stop`. Удаление учебных данных: `docker compose down -v`. После этого OpenBao нужно инициализировать заново, а `secrets/operator/` устареет.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ──TLS :8443──► nginx ──mTLS (lb.client.messenger; API проверяет SAN и отзыв)──► api-1, api-2
|
||||
api ─AMQPS mTLS (EXTERNAL=CN, только publish)─► RabbitMQ ─► worker (только consume)
|
||||
api, worker ──TLS verify-full──► PostgreSQL (роли app_api / app_worker)
|
||||
api, worker ──HTTPS──► MinIO (пользователи messenger-api / messenger-worker, права по префиксам)
|
||||
api ──TLS + ACL──► Valkey (кэш сессий + общий лимит входа)
|
||||
api, worker ──HTTPS + cert auth──► OpenBao (KV: только свой путь, токен 15 мин)
|
||||
api, worker ──mTLS (Ingest CA)──► Collector ──mTLS (Telemetry Clients CA)──► Prometheus / Loki / Tempo ◄── Grafana
|
||||
оператор ──docker compose exec / 127.0.0.1 (HTTPS, mTLS)──► OpenBao, Grafana, Prometheus/Loki/Tempo
|
||||
```
|
||||
|
||||
**Источник истины и транзакции** (как в lab5–lab6):
|
||||
- PostgreSQL — источник истины;
|
||||
- upload в одной транзакции фиксирует вложение, задание и outbox и только потом отвечает 202;
|
||||
- worker публикует результат одной транзакцией;
|
||||
- брокер переносит только ссылки на задания;
|
||||
- OpenBao в бизнес-транзакции не входит: секреты читаются при старте и обновляются фоном.
|
||||
|
||||
**Изменения относительно lab6:**
|
||||
|
||||
| Область | Что сделано | Где |
|
||||
| --- | --- | --- |
|
||||
| PKI | Root CA и два промежуточных: Ingest (только клиенты Collector) и Telemetry Clients (только клиенты хранилищ телеметрии); leaf 90 дн., EC P-256; перевыпуск, отзыв по serial, тестовые просроченный и чужой сертификаты | `scripts/pki.sh` |
|
||||
| Внешний TLS | nginx на `127.0.0.1:8443`, HSTS, TLS 1.2+ | `infra/nginx.conf` |
|
||||
| LB → API | mTLS: nginx предъявляет `lb.client.messenger` и проверяет сервер по `api.internal`. API (`app/serve.py`): CERT_REQUIRED, allowlist DNS SAN, denylist отозванных serial (перечитывается на лету), проверка на каждом соединении. Plaintext-listener нет, health только `127.0.0.1:8081` | `app/serve.py`, `app/main.py` |
|
||||
| Связи с хранилищами | PostgreSQL `verify-full` + `pg_hba` только `hostssl`; MinIO HTTPS; RabbitMQ AMQPS с обязательным клиентским сертификатом; Valkey TLS-only + ACL | `infra/postgres`, `infra/minio`, `infra/rabbitmq`, `infra/valkey`, `app/db.py`, `app/s3.py`, `app/broker.py`, `app/sessions.py` |
|
||||
| Минимальные права | роли БД, S3-политики по префиксам, права RabbitMQ (API: write `^attachments$`; worker: read `^attachments\.process$`; configure ни у кого, топологию создаёт оператор) | `migrations/0007_service_roles.sql`, `infra/minio/policy-*.json`, `definitions.json` |
|
||||
| OpenBao | server mode, хранилище `file` на volume, TLS; cert auth: роль = точный CN, токен 15 мин / max 1 ч; политики «только свой путь»; клиент с renew/re-login и подхватом новой версии KV без рестарта | `infra/openbao/`, `scripts/bao.sh`, `app/openbao.py` |
|
||||
| Телеметрия | Collector принимает только Ingest CA; Prometheus, Loki и Tempo — только Telemetry Clients CA; Grafana обращается к источникам с клиентским сертификатом; scrape-заданий нет; внутренний gRPC и self-метрики только на loopback | `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` |
|
||||
| Аутентификация | Argon2id с параметрами из env (замер ниже); rehash при входе; ≤ 4 одновременных Argon2 на реплику; лимит входа 5 неудач на username и 120 попыток на IP за 60 с — общий для реплик (Valkey), 429 + Retry-After | `app/passwords.py`, `app/main.py`, `app/sessions.py` |
|
||||
| Hardening | все 15 контейнеров non-root, `cap_drop: ALL`, `no-new-privileges`, read-only rootfs + tmpfs, лимиты CPU/RAM/PIDs; нет Docker socket и host network; наружу только `127.0.0.1` | `compose.yaml` |
|
||||
| Обновления по scan | `python:3.12.16-slim`, Pillow 13.0.0, urllib3 2.8.0, nginx 1.30.6, OpenBao 2.6.5; guard использует `scope["path"]` вместо `request.url.path` | `Dockerfile`, `requirements.txt`, `compose.yaml` |
|
||||
|
||||
**Роли, версии, health, volumes, порты, credentials.** Сеть одна — внутренняя сеть Compose. Digest всех образов закреплён в `compose.yaml`. Каждый ключ попадает только в свой контейнер как Docker secret.
|
||||
|
||||
| Роль | Сервис | Образ | Health | Volume | Порт (наружу) | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | `nginx` | nginx:1.30.6-alpine | `curl https://localhost:8443/health/live` | — | `127.0.0.1:8443` | lb-server, lb-client |
|
||||
| API ×2 | `api-1`, `api-2` | сборка, python:3.12.16-slim | loopback `127.0.0.1:8081/health/ready` | — | — | api-N (сервер), api (клиент), api-otel; **паролей нет**, всё из OpenBao |
|
||||
| Worker ×2 | `worker` | та же сборка | heartbeat-файл | — | — | worker (клиент), worker-otel; паролей нет |
|
||||
| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | — | postgres (сервер), пароль superuser (только migrate/оператор) |
|
||||
| S3 | `minio` | madlabcg/minio (digest) | `mc ready` по HTTPS | `miniodata` | — | minio (сервер), root (только minio-init) |
|
||||
| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_running` | `rabbitmqdata` | — | rabbitmq (сервер); пользователей с паролями нет |
|
||||
| Кэш и лимиты | `valkey` | valkey:8.1-alpine | `PING` пользователем `health` | — (кэш) | — | valkey (сервер), пароль ACL `api` |
|
||||
| Секреты | `openbao` | openbao:2.6.5 | `bao status` | `baodata` | — | openbao (сервер); root token и доли у оператора |
|
||||
| Telemetry | `otel-collector`, `prometheus`, `loki`, `tempo` | версии lab6 | promtool / TCP | `promdata`, `lokidata`, `tempodata` | `127.0.0.1:9090/3100/3200` (mTLS operator) | свои серверные и клиентские сертификаты |
|
||||
| UI | `grafana` | grafana:11.5.2 | `curl https://grafana:3000/api/health` | `grafanadata` | `127.0.0.1:3000` (HTTPS + login) | grafana, grafana-out, admin-пароль |
|
||||
|
||||
**Соответствие `.env.example`:**
|
||||
- `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `S3_*_FILE`, `BROKER_URL_FILE` для сервисов больше не используются: эти значения приходят из OpenBao (`messenger/api`, `messenger/worker`);
|
||||
- `BROKER_URL` = `amqps://rabbitmq:5671/messenger?auth=external` — без пароля;
|
||||
- `TLS_CA_FILE`, `TLS_CLIENT_CERT_FILE` и `TLS_CLIENT_KEY_FILE` (вместо `TLS_CERT_FILE`/`TLS_KEY_FILE` из примера), `TLS_SERVER_*`, `BAO_ADDR` задаются в `compose.yaml`;
|
||||
- `ARGON2_*` = 19456 KiB / 2 / 1;
|
||||
- добавлены `PASSWORD_HASH_CONCURRENCY`, `LOGIN_RATE_LIMIT_*`, `BAO_REFRESH_SECONDS` и `BAO_STARTUP_TIMEOUT_SECONDS`.
|
||||
|
||||
**Документы:** модель угроз — [security/THREAT_MODEL.md](security/THREAT_MODEL.md) (14 угроз с тестами и разбор scan); матрица доверия — [contracts/trust-matrix.md](contracts/trust-matrix.md); инструкция оператора OpenBao — [infra/openbao/README.md](infra/openbao/README.md).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| 3: прежние возможности через HTTPS с проверкой CA и SAN | `make check`, `make validate`, `make test`, `make test-security` | все PASS; пустой trust store → ошибка проверки сертификата | `evidence/smoke_check_validate_security.txt`, `evidence/smoke.txt` |
|
||||
| 3: persistence через HTTPS | `make test-persistence` (hook перезапускает все хранилища, включая OpenBao с unseal, и сервисы) | PASS; uptime всех контейнеров сброшен | `evidence/persistence.txt` |
|
||||
| 3: отказ реплики через HTTPS | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 1.27 и 0.67 с | `evidence/failover.txt` |
|
||||
| 3: минимум lab5–lab6 | `test_worker_down.py`, `test_failure_windows.py` (A–I), `variants_check.py`, `test_observability.py` | все PASS | `evidence/lab6_worker_down.txt`, `evidence/lab5_failure_windows.txt`, `evidence/variants_limits.txt`, `evidence/observability.txt` |
|
||||
| 3: Argon2id ≥ 19 MiB, t=2, p=1, параметры обоснованы замером | `login_budget.py` (bench в контейнере API, 0.5 CPU); `test_auth_acl.py` | PHC-строка `$argon2id$v=19$m=19456,t=2,p=1$…` с солью; 11 мс на хеш; таблица ниже | `evidence/login_budget.json`, `evidence/auth_acl.txt` |
|
||||
| 3: LB → API mTLS, отказ без сертификата, с чужим CA, с неверной identity | аудит A | 14/14: без сертификата, чужой CA с именем lb, `intruder`/`worker`/`api` от того же CA → соединение закрыто; `lb` → 200; plaintext → нет ответа | `evidence/tls_audit_final.txt` |
|
||||
| 3: нет обхода публичным портом | аудит A, K; `check_hardening.py` | порты API не опубликованы; health 8081 закрыт из сети | `evidence/tls_audit_side_channels.txt`, `evidence/hardening.txt` |
|
||||
| 3: TLS с проверкой сервера на всех связях, раздельные учётные данные | аудит B, C, D, E, F, G, H, I, J | чужой CA как trust anchor, неверный hostname, plaintext → отказ на каждой связи | `evidence/tls_audit_final.txt` |
|
||||
| 3: OpenBao не dev-mode, persistent, сервисная аутентификация, scoped policy, без root token в app | аудит H; `bao.sh`; persistence | cert auth: только своя роль, токен ≤ 15 мин; чужой путь, запись, политика, admin-пути → 403; данные пережили рестарт OpenBao | `evidence/tls_audit_final.txt`, `evidence/persistence.txt` |
|
||||
| 3: контейнеры без privileged/host network/socket, non-root, без лишних capabilities, лимиты; наружу только нужное | `check_hardening.py` (`docker inspect`) | 15 контейнеров, 0 проблем; опубликованы только 8443, 3000, 9090, 3100, 3200 на 127.0.0.1 | `evidence/hardening.txt`, `evidence/hardening.json` |
|
||||
| 3: секреты не в Git, образах, telemetry | Trivy secret scan; `test_observability.py` | в слоях образа секретов 0; пароль, токен, текст, filename, Authorization, PNG в трассах и логах не найдены | `evidence/image_scan.txt`, `evidence/observability.txt` |
|
||||
| 3: лимит входа 429 + Retry-After, размер запросов, посторонний, logout/expiry | `test_auth_acl.py` | 404 постороннему на всё; поддельный X-User-Id игнорируется; logout и expiry → 401 на обеих репликах; 70 kB JSON и > 10 MiB upload → 413; лимит входа → 429 | `evidence/auth_acl.txt` |
|
||||
| 3: локальный bootstrap сертификатов | `scripts/pki.sh init` | CA и leaf создаются в `secrets/pki/`, ключи не в Git | `scripts/pki.sh` |
|
||||
| 4: mTLS к брокеру и Collector, разные identities, чужой сертификат того же CA не даёт прав | аудит D, E | RabbitMQ: `intruder`, `lb` (тот же CA) → ACCESS_REFUSED; API не может consume и declare; worker не может publish. Collector: `lb`, `api` (Root), `grafana-out` → отказ; api-otel, worker-otel → 200 | `evidence/tls_audit_final.txt` |
|
||||
| 4: хеши и ошибки не раскрывают credentials; rehash; бюджет login DoS; общий rate limit | `test_auth_acl.py`, `login_budget.py` | старый хеш `m=8192,t=1` пересчитан в `m=19456,t=2` при входе; 5 неудач на api-1 и api-2 вместе → 429 даже с верным паролем; тело 429 без деталей | `evidence/auth_acl.txt`, `evidence/login_budget.json` |
|
||||
| 4: ротация одного рабочего secret без пересборки | `test_secret_rotation.py` (оператор) | пароль `app_worker` изменён в PostgreSQL и OpenBao; оба worker переключились за 10 с; после принудительного переподключения задания обрабатываются; старый пароль → `password authentication failed`; 0 ошибок, 0 упавших заданий | `evidence/secret_rotation.txt` |
|
||||
| 4: ротация leaf без пересборки, отзыв старой identity | `test_cert_rotation.py` | `lb-client` перевыпущен (serial …1418 → …142A), `nginx -s reload` за 1.4 с; после отзыва старый сертификат закрыт на обеих репликах (`reason=revoked`); 161 запрос, 0 ошибок | `evidence/cert_rotation.txt` |
|
||||
| 4: ограниченное поведение при expiry и sealed OpenBao | `test_bao_sealed.py` (оператор), `test_cert_rotation.py` | sealed: smoke проходит, сервисы держат секреты, перезапущенный worker ждёт и не стартует без секретов, после unseal здоров через 16 с. Просроченный сертификат с верной identity отклоняется | `evidence/bao_sealed.txt`, `evidence/cert_rotation.txt` |
|
||||
| 4: bootstrap trust и renewal описаны, нет постоянного «секрета для секретов» | `app/openbao.py`, модель угроз | первичная identity — TLS-ключ сервиса; токен 15 мин, renew/re-login каждые 15 с | `security/THREAT_MODEL.md`, `infra/openbao/README.md` |
|
||||
| 4: read_only, no-new-privileges, drop capabilities, tmpfs, исключения объяснены | `check_hardening.py` | read-only rootfs у **всех** 15 контейнеров, включая stateful; запись только в volumes и tmpfs (для non-root nginx tmpfs смонтированы с его uid). Исключений не понадобилось | `evidence/hardening.txt` |
|
||||
| 4: dependency/image scan с разбором применимости | Trivy по образу приложения, nginx, OpenBao | приложение: CRITICAL ОС 9 → 2 (исправлений нет, неприменимы), Python CRITICAL 5 → 0, HIGH 43 → 3 (starlette, неприменимы); nginx и OpenBao обновлены | `evidence/image_scan.txt`, `security/THREAT_MODEL.md` (раздел Scan) |
|
||||
| 4: ≥ 6 угроз с контролями и воспроизводимыми тестами | модель угроз | 14 угроз T1–T14, у каждой отрицательный тест | `security/THREAT_MODEL.md` |
|
||||
| 4: upload-worker ограничен по CPU/RAM/времени | `variants_check.py` | дочерний процесс с RLIMIT_AS/CPU и таймаутом; изображение 39.5 Мпикс — пик RSS 337 MiB; лимиты пикселей соблюдены | `evidence/variants_limits.txt` |
|
||||
| 4: TLS-аудит всей матрицы, ACL, отсутствие утечек | `tls_audit.py`, `test_auth_acl.py`, `test_observability.py` | 135 проверок, 0 провалов (A–K); ACL и приватность — выше | `evidence/tls_audit_final.txt` |
|
||||
| Внутренняя телеметрия и Grafana работают через mTLS | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, Loki и Tempo выполнены через Grafana | `evidence/dashboard_check.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, OpenBao 0.5 CPU / 256 MiB, nginx 0.25 CPU / 64 MiB. Версии — в таблице ролей.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
### Параметры Argon2id (замер в контейнере API, лимит 0.5 CPU, 10 хешей)
|
||||
|
||||
| m, KiB | t | p | p50, мс | max, мс | Пик RSS, MiB |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| **19456** | **2** | **1** | **11.3** | 66.6 | 37 |
|
||||
| 12288 | 3 | 1 | 9.3 | 62.0 | 37 |
|
||||
| 47104 | 1 | 1 | 16.6 | 69.2 | 83 |
|
||||
| 65536 | 3 | 1 | 171.6 | 175.5 | 101 |
|
||||
| 65536 | 3 | 4 | 390.0 | 806.1 | 101 |
|
||||
|
||||
**Выбор: m = 19 MiB, t = 2, p = 1** (минимум OWASP).
|
||||
- При 4 параллельных слотах на реплику это ~80 MiB памяти и ~11 мс CPU на вход. Обе реплики выдерживают ~40 входов/с, оставаясь в лимите 512 MiB.
|
||||
- Вариант `m = 64 MiB, t = 3` дороже в 15 раз по времени: 4 слота заняли бы ~400 MiB из 512 (риск OOM), а пропускная способность входа упала бы до ~3/с на реплику. Это сделало бы DoS через вход проще.
|
||||
- Замер — на этом ноутбуке. CFS разрешает кратковременно превышать квоту CPU, поэтому под длительной нагрузкой время растёт (см. ниже).
|
||||
|
||||
### Бюджет защиты от DoS через вход (`login_budget.py`, 30 с на фазу)
|
||||
|
||||
Модель распределённого атакующего: лимит по IP поднят на время замера, у каждой попытки новый несуществующий username. Параллельно идут обычные запросы (POST сообщения, 5 в секунду).
|
||||
|
||||
| Фаза | Попыток входа/с | Латентность 401, p50 / p99 | CPU обеих API, % ядра | RSS API | Обычные запросы: ошибки, p50 / p99 |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| без атаки | 0 | — | 10.3 | 97 MiB | 0, 9.5 / 46.4 мс |
|
||||
| 32 атакующих | 40.4 | 803 / 2397 мс | 89.6 (лимит 100) | 183–197 MiB | 0, 28.5 / 295 мс |
|
||||
|
||||
Вывод: атака упирается в лимит CPU, но память ограничена слотами Argon2 (рост на ~100 MiB), а обычный трафик продолжает обслуживаться без ошибок, хотя p99 растёт в 6 раз. Отказов 503 «hash_busy» не было, очередь к слоту укладывалась в 3 с. С одного IP атакующий получает 429 после 107–120 попыток в минуту (`auth_acl.txt`).
|
||||
|
||||
### Отказы и ротации
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Ротация leaf `lb-client` под нагрузкой 10 запросов/с | без пересборки и простоя; старый отозван | `pki.sh issue` → `nginx -s reload` (1.4 с) → старый и новый действуют → `revoke-serial` → старый отклонён на api-1 и api-2 через ~1 с, новый 200, просроченный отклонён | 161/161 запросов успешно, smoke после — PASS |
|
||||
| Ротация пароля `app_worker` под трафиком (5 запросов/с + 1 upload/с) | worker переключается без рестарта, старый пароль недействителен | новая версия KV подхвачена обоими worker за 10 с; сессии `app_worker` принудительно завершены, переподключение с новым паролем; 3 upload → ready | 116 запросов, 20 upload, 0 ошибок, 0 упавших заданий |
|
||||
| OpenBao запечатан на 58 с | работающие сервисы продолжают, новые ждут, ничего не ослабляется | smoke PASS при запечатанном OpenBao; `openbao_unavailable` в логах; перезапущенный worker-2 в состоянии `starting`, ждёт OpenBao; upload обработан вторым worker | после unseal worker-2 healthy через 16 с |
|
||||
| Рестарт всех хранилищ, включая OpenBao (hook persistence) | данные и сессии сохраняются | PASS; OpenBao распечатан оператором внутри hook | потерь нет |
|
||||
| Остановка api-1, затем api-2 (failover) | запись восстанавливается ≤ 10 с, без дублей | PASS ×2: 128 и 130 записей, 0 неудачных попыток | 1.27 и 0.67 с |
|
||||
| Обновление OpenBao 2.1.0 → 2.6.5 (по scan) | хранилище `file` подхватывается | версия 2.6.5 стартовала на старом volume, данные на месте; после unseal вход сервисов работает | требуется unseal (как после любого рестарта) |
|
||||
|
||||
### Что нашли и исправили по ходу проверок
|
||||
|
||||
- **Valkey ACL** не разрешал `INCRBY`, хотя valkey-py вызывает её для `INCR`: общий лимит входа молча откатился бы на локальный. Найдено аудитом G.
|
||||
- **Erlang distribution RabbitMQ** (25672) и **кластерный порт OpenBao 2.6** (8201) были доступны из сети. Привязаны к loopback и отключены соответственно (аудит K).
|
||||
- **Self-метрики Collector** нельзя было отправить по OTLP с клиентским сертификатом. Переведены на loopback-сбор внутри процесса, plaintext-scrape убран совсем.
|
||||
- **Self-метрики Collector** приходили с `service.instance.id` = случайный UUID (новый после каждого рестарта), это давало рост кардинальности меток. Задано стабильное значение.
|
||||
- **У worker не было лимита PIDs** (`check_hardening.py`).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**Выдерживается** (проверено): отказ реплики API; рестарт всех хранилищ с volumes; запечатанный или перезапущенный OpenBao (без простоя работающих сервисов); недоступная телеметрия (из lab6); компрометация отдельной identity — её права ограничены (аудит), сертификат LB отзывается.
|
||||
|
||||
**Не выдерживается и не заявляется:** потеря Docker-host и компрометация хоста или Docker daemon (root на хосте читает `secrets/`); высокая доступность единственных PostgreSQL, nginx, OpenBao и Collector.
|
||||
|
||||
**Учебные упрощения и известные ограничения:**
|
||||
- Ключи CA, все 3 доли unseal и root token — у одного оператора на хосте стенда (`secrets/`, вне Git, 0600/0700). В production — разные держатели, HSM/KMS, отзыв root token.
|
||||
- Отзыв по serial проверяет только API. RabbitMQ, OpenBao и Collector отзыв не проверяют; там identity отзывается удалением пользователя или роли.
|
||||
- Collector не связывает `service.name` с сертификатом. В Prometheus, Loki и Tempo нет прав по identity: любой клиент Telemetry Clients CA и пишет, и читает (CA выдан только collector, grafana, operator).
|
||||
- Истечение токена OpenBao (15 мин) не выжидалось в тесте. Продление и повторный вход видны в логах (`openbao_login`), истечение сертификата проверено просроченным сертификатом.
|
||||
- Алерты lab6 (сценарии `test_incidents.py`) в lab7 заново не прогонялись. Проверены трассы, логи, приватность и все запросы dashboard через mTLS. Правила алертов загружены (`promtool check rules`: 13 правил).
|
||||
- Исчерпание пула БД переводит реплику в not ready (из lab6). Отдельный пул для readiness не реализован.
|
||||
- `FAULT_INJECTION=1` и `X-Fault` допустимы только на учебном стенде. TCP-healthcheck Loki и Tempo оставляет в их логах записи `TLS handshake error … EOF`.
|
||||
- Starlette 0.41.3 не обновлялась: применимых уязвимостей не осталось, а обновление требует мажорного перехода FastAPI.
|
||||
- Первый запуск PKI и OpenBao — ручные шаги оператора; они воспроизводимы скриптами.
|
||||
|
||||
**Следующие шаги:** короткоживущие сертификаты из OpenBao PKI с автоматическим renewal (трек 2), проверка отзыва во всех сервисах (CRL/OCSP), отдельный пул для readiness, обновление FastAPI и Starlette.
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""RabbitMQ topology and confirmed publishing (aio-pika).
|
||||
|
||||
Topology (lab7: declared by the operator in infra/rabbitmq/definitions.json; services only check it passively):
|
||||
exchange attachments direct, durable
|
||||
queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit
|
||||
exchange attachments.dlx fanout, durable
|
||||
queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator)
|
||||
Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker
|
||||
has taken responsibility for the message (written to the quorum log).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType, Message
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.broker")
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
|
||||
|
||||
def tls_context() -> ssl.SSLContext:
|
||||
"""lab7: AMQPS with mutual TLS. The broker verifies our client certificate and maps its CN to the RabbitMQ user
|
||||
(auth=external): api.client.messenger may only publish, worker.client.messenger may only consume."""
|
||||
ctx = ssl.create_default_context(cafile=config.TLS_CA_FILE)
|
||||
ctx.load_cert_chain(config.TLS_CLIENT_CERT_FILE, config.TLS_CLIENT_KEY_FILE)
|
||||
return ctx
|
||||
|
||||
|
||||
async def connect():
|
||||
return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, ssl_context=tls_context(),
|
||||
client_properties={"connection_name": config.INSTANCE_ID})
|
||||
|
||||
|
||||
async def get_exchange(channel):
|
||||
"""Passive: the topology (exchanges, quorum queues, DLX, bindings) is created by the operator from
|
||||
infra/rabbitmq/definitions.json. Services have NO configure permission, so they only check that it exists."""
|
||||
return await channel.get_exchange(config.BROKER_EXCHANGE, ensure=True)
|
||||
|
||||
|
||||
async def get_queue(channel):
|
||||
return await channel.get_queue(config.BROKER_QUEUE, ensure=True)
|
||||
|
||||
|
||||
class Publisher:
|
||||
def __init__(self) -> None:
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.exchange = None
|
||||
|
||||
async def ensure(self) -> None:
|
||||
if self.channel is not None and not self.channel.is_closed:
|
||||
return
|
||||
if self.connection is None or self.connection.is_closed:
|
||||
self.connection = await connect()
|
||||
self.channel = await self.connection.channel(publisher_confirms=True)
|
||||
self.exchange = await get_exchange(self.channel)
|
||||
|
||||
async def publish(self, body: dict) -> None:
|
||||
await self.ensure()
|
||||
message = Message(
|
||||
json.dumps(body, separators=(",", ":")).encode(),
|
||||
content_type="application/json",
|
||||
delivery_mode=DeliveryMode.PERSISTENT,
|
||||
message_id=body["job_id"],
|
||||
type=body["type"],
|
||||
# Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling.
|
||||
headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}},
|
||||
)
|
||||
# mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack.
|
||||
await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5)
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
if self.connection is not None:
|
||||
await self.connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.connection = self.channel = self.exchange = None
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
# --- lab7: TLS material (paths only) and secrets from OpenBao ------------------------------------------------
|
||||
TLS_CA_FILE = os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt")
|
||||
TLS_CLIENT_CERT_FILE = os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt")
|
||||
TLS_CLIENT_KEY_FILE = os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key")
|
||||
DATABASE_SSLMODE = os.environ.get("DATABASE_SSLMODE", "verify-full")
|
||||
BAO_REFRESH_SECONDS = _float("BAO_REFRESH_SECONDS", 15.0)
|
||||
SECRETS = {}
|
||||
if os.environ.get("BAO_ADDR"):
|
||||
from app import openbao as _openbao
|
||||
|
||||
SECRETS = _openbao.load(_float("BAO_STARTUP_TIMEOUT_SECONDS", 120.0))
|
||||
|
||||
|
||||
def _from_bao(key, env_name, default=""):
|
||||
"""OpenBao value when the service runs with BAO_ADDR, otherwise the old *_FILE / env (tools, tests)."""
|
||||
return SECRETS.get(key) or _secret(env_name, default)
|
||||
|
||||
|
||||
DATABASE_URL = _from_bao("database_url", "DATABASE_URL")
|
||||
CACHE_URL = _from_bao("cache_url", "CACHE_URL")
|
||||
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
|
||||
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
|
||||
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
|
||||
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
|
||||
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
|
||||
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
|
||||
READY_FAILURES = _int("READY_FAILURES", 2)
|
||||
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
|
||||
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
SESSION_REVOKED_TTL_SECONDS = max(
|
||||
_int("SESSION_REVOKED_TTL_SECONDS", 60),
|
||||
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
|
||||
)
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = _from_bao("s3_access_key_id", "S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = _from_bao("s3_secret_access_key", "S3_SECRET_ACCESS_KEY")
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
|
||||
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
|
||||
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
|
||||
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
|
||||
|
||||
# --- lab5: broker, jobs, worker ---
|
||||
BROKER_URL = _secret("BROKER_URL") # amqps://...?auth=external: identity = client certificate, no password
|
||||
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
|
||||
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
||||
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
|
||||
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
|
||||
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20)
|
||||
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
|
||||
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2)
|
||||
WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY
|
||||
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
|
||||
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60)
|
||||
JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20)
|
||||
JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0)
|
||||
JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0)
|
||||
JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0)
|
||||
WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768)
|
||||
WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25)
|
||||
WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
|
||||
OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0)
|
||||
OUTBOX_BATCH = _int("OUTBOX_BATCH", 50)
|
||||
OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400)
|
||||
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
|
||||
# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables).
|
||||
MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200)
|
||||
PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0)
|
||||
THUMBNAIL_SIZE = 256
|
||||
CROP_SIZE = 128
|
||||
OPTIMIZED_SIZE = 1280
|
||||
JPEG_QUALITY = _int("JPEG_QUALITY", 85)
|
||||
WEBP_QUALITY = _int("WEBP_QUALITY", 80)
|
||||
# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py).
|
||||
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
|
||||
|
||||
# --- lab7: authentication hardening ---------------------------------------------------------------------------
|
||||
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456)
|
||||
ARGON2_ITERATIONS = _int("ARGON2_ITERATIONS", 2)
|
||||
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
|
||||
PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4)
|
||||
PASSWORD_HASH_QUEUE_SECONDS = _float("PASSWORD_HASH_QUEUE_SECONDS", 3.0)
|
||||
LOGIN_RATE_LIMIT_PER_USER = _int("LOGIN_RATE_LIMIT_PER_USER", 5)
|
||||
LOGIN_RATE_LIMIT_PER_IP = _int("LOGIN_RATE_LIMIT_PER_IP", 120)
|
||||
LOGIN_RATE_WINDOW_SECONDS = _int("LOGIN_RATE_WINDOW_SECONDS", 60)
|
||||
# mTLS on the API listener: only these client identities (DNS SAN) may connect; revoked serials are refused.
|
||||
ALLOWED_CLIENT_IDENTITIES = frozenset(
|
||||
x.strip() for x in os.environ.get("ALLOWED_CLIENT_IDENTITIES", "lb.client.messenger").split(",") if x.strip())
|
||||
REVOKED_SERIALS_FILE = os.environ.get("REVOKED_SERIALS_FILE", "")
|
||||
TLS_SERVER_CERT_FILE = os.environ.get("TLS_SERVER_CERT_FILE", "")
|
||||
TLS_SERVER_KEY_FILE = os.environ.get("TLS_SERVER_KEY_FILE", "")
|
||||
HEALTH_LISTEN_PORT = _int("HEALTH_LISTEN_PORT", 8081)
|
||||
GRACEFUL_TIMEOUT = _int("GRACEFUL_TIMEOUT", 8)
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
|
||||
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
class BadCursor(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fmt(ts: datetime) -> str:
|
||||
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
|
||||
|
||||
|
||||
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
|
||||
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(scope: str, cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
raise BadCursor() from None
|
||||
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
|
||||
raise BadCursor()
|
||||
ts, item_id = data["t"], data["i"]
|
||||
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
|
||||
raise BadCursor()
|
||||
try:
|
||||
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
|
||||
if str(uuid.UUID(item_id)) != item_id:
|
||||
raise BadCursor()
|
||||
except ValueError:
|
||||
raise BadCursor() from None
|
||||
if encode_cursor(scope, ts, item_id) != cursor:
|
||||
raise BadCursor()
|
||||
return parsed, item_id
|
||||
@@ -0,0 +1,145 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from app import config, telemetry
|
||||
from app.errors import unavailable
|
||||
|
||||
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
|
||||
OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)")
|
||||
TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE)
|
||||
POOL_WAIT_SPAN_SECONDS = 0.005
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
|
||||
def statement(query) -> tuple:
|
||||
"""Span name and attributes for a parametrised SQL statement (values are never part of the text)."""
|
||||
text = query if isinstance(query, str) else str(query)
|
||||
op = OPERATION_RE.match(text)
|
||||
table = TABLE_RE.search(text)
|
||||
operation = op.group(1).upper() if op else "SQL"
|
||||
name = operation + (" " + table.group(1) if table else "")
|
||||
return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger",
|
||||
"db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]}
|
||||
|
||||
|
||||
async def discard(conn) -> None:
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Guarded:
|
||||
def __init__(self, conn, timeout: float):
|
||||
self.conn = conn
|
||||
self.timeout = timeout
|
||||
|
||||
async def execute(self, *args, **kwargs):
|
||||
# Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included).
|
||||
# (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.)
|
||||
name, attrs = statement(args[0] if args else "")
|
||||
with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span:
|
||||
try:
|
||||
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(self.conn)
|
||||
span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout))
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
def transaction(self):
|
||||
return self.conn.transaction()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self) -> None:
|
||||
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS * 2,
|
||||
)
|
||||
self.pool = AsyncConnectionPool(
|
||||
conninfo=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
timeout=config.DB_POOL_TIMEOUT,
|
||||
max_lifetime=600.0,
|
||||
open=False,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": config.DB_CONNECT_TIMEOUT,
|
||||
"options": options,
|
||||
# lab7: TLS with full server verification (CA + hostname "postgres"); no fallback to plaintext.
|
||||
"sslmode": config.DATABASE_SSLMODE,
|
||||
"sslrootcert": config.TLS_CA_FILE,
|
||||
},
|
||||
)
|
||||
|
||||
def rotate(self, conninfo: str) -> None:
|
||||
"""New credentials from OpenBao: new connections use them; open ones keep working until max_lifetime."""
|
||||
if conninfo and conninfo != self.pool.conninfo:
|
||||
self.pool.conninfo = conninfo
|
||||
log.info("db_credentials_rotated: new connections use the new password")
|
||||
|
||||
async def open(self) -> None:
|
||||
await self.pool.open(wait=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.pool.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
started, acquired = time.time_ns(), False
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
|
||||
acquired = True
|
||||
waited = (time.time_ns() - started) / 1e9
|
||||
if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans
|
||||
telemetry.tracer.start_span("db.pool.acquire", start_time=started,
|
||||
attributes={"db.system": "postgresql",
|
||||
"db.client.connection.wait_s": round(waited, 4)}).end()
|
||||
yield Guarded(conn, config.DB_OP_TIMEOUT)
|
||||
except BROKEN as exc:
|
||||
if not acquired:
|
||||
span = telemetry.tracer.start_span("db.pool.acquire", start_time=started,
|
||||
attributes={"db.system": "postgresql"})
|
||||
span.record_exception(exc)
|
||||
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
span.end()
|
||||
raise unavailable("Хранилище недоступно") from None
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
async with self.connection() as conn:
|
||||
async with conn.transaction():
|
||||
yield conn
|
||||
|
||||
async def in_transaction(self, action):
|
||||
async def run():
|
||||
async with self.transaction() as conn:
|
||||
return await action(conn)
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
|
||||
try:
|
||||
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(conn)
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,37 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> ApiError:
|
||||
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
|
||||
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
|
||||
|
||||
|
||||
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||||
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off).
|
||||
|
||||
A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`.
|
||||
Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata.
|
||||
Crash faults fire only on the first attempt and first delivery, so recovery can be observed.
|
||||
api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish
|
||||
worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT
|
||||
worker_crash_after_commit worker exits after the result COMMIT, before broker ack
|
||||
worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1
|
||||
transientN attempts 1..N fail with a transient error (N = 1..9)
|
||||
permanent permanent error on the first attempt -> failed without retries
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.faults")
|
||||
TRANSIENT_RE = re.compile(r"fault-transient([1-9])")
|
||||
|
||||
|
||||
def active(filename: Optional[str], name: str) -> bool:
|
||||
return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename
|
||||
|
||||
|
||||
def transient_until(filename: Optional[str]) -> int:
|
||||
if not config.FAULT_INJECTION or not filename:
|
||||
return 0
|
||||
match = TRANSIENT_RE.search(filename)
|
||||
return int(match.group(1)) if match else 0
|
||||
|
||||
|
||||
def crash(point: str) -> None:
|
||||
log.error("FAULT INJECTION: process exits at %s", point)
|
||||
logging.shutdown()
|
||||
os._exit(70)
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
|
||||
|
||||
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
|
||||
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
|
||||
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
|
||||
only a warning. Both finish before anything is stored.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app import config
|
||||
from app.errors import ApiError, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.imagecheck")
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
|
||||
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||
FULL_DECODE_MAX_PIXELS = 4000000
|
||||
SCAN_SECONDS = 10.0
|
||||
OUT_STEP = 1 << 20
|
||||
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
|
||||
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
|
||||
jpeg_slot = threading.Lock()
|
||||
|
||||
|
||||
class Invalid(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def invalid() -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
|
||||
|
||||
|
||||
class RowPlan:
|
||||
def __init__(self, segments):
|
||||
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
|
||||
self.seg = 0
|
||||
self.done = 0
|
||||
self.pos = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
i, n = 0, len(data)
|
||||
while i < n:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
if self.seg >= len(self.segments):
|
||||
raise Invalid("data after the last row")
|
||||
length, rows = self.segments[self.seg]
|
||||
if self.pos > 0:
|
||||
take = min(length - self.pos, n - i)
|
||||
i += take
|
||||
self.pos += take
|
||||
if self.pos == length:
|
||||
self.pos = 0
|
||||
self.done += 1
|
||||
continue
|
||||
full = min((n - i) // length, rows - self.done)
|
||||
if full > 0:
|
||||
if max(data[i:i + full * length:length]) > 4:
|
||||
raise Invalid("bad filter type")
|
||||
i += full * length
|
||||
self.done += full
|
||||
else:
|
||||
if data[i] > 4:
|
||||
raise Invalid("bad filter type")
|
||||
self.pos = n - i
|
||||
i = n
|
||||
|
||||
def complete(self) -> bool:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
return self.seg >= len(self.segments) and self.pos == 0
|
||||
|
||||
|
||||
def png_plan(width, height, depth, color, interlace):
|
||||
bits = CHANNELS[color] * depth
|
||||
if not interlace:
|
||||
return RowPlan([(1 + (width * bits + 7) // 8, height)])
|
||||
segments = []
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if w and h:
|
||||
segments.append((1 + (w * bits + 7) // 8, h))
|
||||
return RowPlan(segments)
|
||||
|
||||
|
||||
def scan_png(fileobj) -> None:
|
||||
deadline = time.monotonic() + SCAN_SECONDS
|
||||
fileobj.seek(0)
|
||||
if fileobj.read(8) != PNG_SIGNATURE:
|
||||
raise Invalid("signature")
|
||||
plan = decomp = None
|
||||
palette = seen_idat = idat_closed = False
|
||||
color = None
|
||||
first = True
|
||||
while True:
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise Invalid("truncated before IEND")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise Invalid("chunk length")
|
||||
if first and kind != b"IHDR":
|
||||
raise Invalid("IHDR must be first")
|
||||
crc = zlib.crc32(kind)
|
||||
if kind == b"IHDR":
|
||||
body = fileobj.read(length)
|
||||
if not first or length != 13 or len(body) != 13:
|
||||
raise Invalid("IHDR")
|
||||
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
|
||||
raise Invalid("IHDR fields")
|
||||
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
|
||||
raise Invalid("color type / depth")
|
||||
plan = png_plan(width, height, depth, color, interlace)
|
||||
decomp = zlib.decompressobj()
|
||||
crc = zlib.crc32(body, crc)
|
||||
else:
|
||||
remaining = length
|
||||
while remaining:
|
||||
buf = fileobj.read(min(remaining, 65536))
|
||||
if not buf:
|
||||
raise Invalid("truncated chunk")
|
||||
remaining -= len(buf)
|
||||
crc = zlib.crc32(buf, crc)
|
||||
if kind == b"IDAT":
|
||||
if idat_closed or (color == 3 and not palette):
|
||||
raise Invalid("IDAT order")
|
||||
seen_idat = True
|
||||
out = decomp.decompress(buf, OUT_STEP)
|
||||
while True:
|
||||
plan.feed(out)
|
||||
if not decomp.unconsumed_tail:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
|
||||
if kind == b"PLTE":
|
||||
palette = True
|
||||
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
||||
raise Invalid("PLTE")
|
||||
elif kind != b"IDAT" and seen_idat:
|
||||
idat_closed = True
|
||||
tail = fileobj.read(4)
|
||||
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
|
||||
raise Invalid("crc")
|
||||
first = False
|
||||
if kind == b"IEND":
|
||||
if length or not seen_idat:
|
||||
raise Invalid("IEND")
|
||||
break
|
||||
if not decomp.eof or decomp.unused_data or not plan.complete():
|
||||
raise Invalid("pixel data does not match IHDR")
|
||||
|
||||
|
||||
def check_jpeg(fileobj) -> None:
|
||||
fileobj.flush()
|
||||
descriptor = fileobj.fileno()
|
||||
os.lseek(descriptor, 0, os.SEEK_SET)
|
||||
try:
|
||||
with jpeg_slot:
|
||||
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=SCAN_SECONDS, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("jpeg_check_timeout")
|
||||
raise unavailable("Проверка изображения не успела завершиться") from None
|
||||
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
|
||||
if done.returncode < 0:
|
||||
log.error("jpeg_check_killed signal=%d", -done.returncode)
|
||||
raise unavailable("Проверка изображения прервана")
|
||||
if done.returncode != 0:
|
||||
raise invalid()
|
||||
|
||||
|
||||
def inspect_image(fileobj, size: int, content_type: str) -> None:
|
||||
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
img = Image.open(fileobj)
|
||||
width, height = img.size
|
||||
if img.format != FORMATS[content_type]:
|
||||
raise invalid()
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
|
||||
if content_type == "image/png":
|
||||
scan_png(fileobj)
|
||||
if width * height <= FULL_DECODE_MAX_PIXELS:
|
||||
fileobj.seek(0)
|
||||
Image.open(fileobj).load()
|
||||
else:
|
||||
check_jpeg(fileobj)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise invalid() from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a
|
||||
wall-clock timeout, so a hostile or huge image can only kill this process, never the worker.
|
||||
|
||||
usage: python -m app.imageproc <source> <out_dir> <max_pixels>
|
||||
stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex.
|
||||
|
||||
Rules (contracts/README.md "Фоновая обработка"):
|
||||
thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px
|
||||
crop JPEG, centred square scaled to 128x128 (small sources are upscaled)
|
||||
optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4
|
||||
EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha).
|
||||
EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB).
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageFile, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
THUMB, CROP, OPTIMIZED = 256, 128, 1280
|
||||
JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85"))
|
||||
WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80"))
|
||||
ORIENTATION_TAG = 0x0112
|
||||
EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5
|
||||
|
||||
|
||||
def fit(width: int, height: int, bound: int):
|
||||
"""Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales."""
|
||||
if width <= bound and height <= bound:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return bound, max(1, height * bound // width)
|
||||
return max(1, width * bound // height), bound
|
||||
|
||||
|
||||
def has_alpha(img) -> bool:
|
||||
return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info)
|
||||
|
||||
|
||||
def to_8bit(img):
|
||||
if img.mode in ("I;16", "I;16B", "I;16L", "I"):
|
||||
return img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
|
||||
return img
|
||||
|
||||
|
||||
def on_white(img):
|
||||
if has_alpha(img):
|
||||
rgba = img.convert("RGBA")
|
||||
flat = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
flat.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return flat, rgba
|
||||
return img.convert("RGB"), None
|
||||
|
||||
|
||||
def process(source: str, out_dir: str, max_pixels: int) -> list:
|
||||
img = Image.open(source)
|
||||
if img.format not in ("PNG", "JPEG"):
|
||||
raise ValueError("format")
|
||||
width, height = img.size
|
||||
if width * height > max_pixels:
|
||||
sys.exit(EXIT_TOO_LARGE)
|
||||
orientation = img.getexif().get(ORIENTATION_TAG, 1)
|
||||
upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height)
|
||||
thumb_size = fit(*upright, THUMB)
|
||||
opt_size = fit(*upright, OPTIMIZED)
|
||||
if img.format == "JPEG":
|
||||
# DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept).
|
||||
need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0])
|
||||
img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1])))
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = to_8bit(img)
|
||||
flat, rgba = on_white(img)
|
||||
del img
|
||||
|
||||
results = []
|
||||
|
||||
def save(name, image, ext, fmt, content_type, **params):
|
||||
path = os.path.join(out_dir, name + "." + ext)
|
||||
image.save(path, fmt, **params)
|
||||
results.append({"name": name, "file": path, "content_type": content_type,
|
||||
"width": image.size[0], "height": image.size[1]})
|
||||
|
||||
jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False}
|
||||
save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0),
|
||||
"jpg", "JPEG", "image/jpeg", **jpeg)
|
||||
w, h = flat.size
|
||||
side = min(w, h)
|
||||
left, top = (w - side) // 2, (h - side) // 2
|
||||
save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)),
|
||||
"jpg", "JPEG", "image/jpeg", **jpeg)
|
||||
source_for_webp = rgba if rgba is not None else flat
|
||||
optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize(
|
||||
opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3])
|
||||
try:
|
||||
results = process(source, out_dir, max_pixels)
|
||||
except MemoryError:
|
||||
sys.exit(EXIT_TOO_COMPLEX)
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
sys.exit(EXIT_INVALID)
|
||||
json.dump(results, sys.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker.
|
||||
|
||||
State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it
|
||||
(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes
|
||||
`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT.
|
||||
|
||||
enqueue nothing -> queued (API, same txn as attachment 'queued')
|
||||
claim queued(due) -> processing, lease (worker; also takes over an expired lease)
|
||||
heartbeat processing: lease_until += lease (only while owner is still alive)
|
||||
complete processing -> succeeded + variants (fenced by lease_owner)
|
||||
fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts)
|
||||
sweep_leases processing(expired lease) -> queued | failed
|
||||
replay failed -> queued (explicit operator action, audited)
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from app import config, telemetry
|
||||
from app.cursor import fmt
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
SCHEMA_VERSION = 1
|
||||
VARIANT_ORDER = ("thumbnail", "crop", "optimized")
|
||||
REQUIRED_VARIANTS = VARIANT_ORDER
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int) -> float:
|
||||
"""Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd)."""
|
||||
ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)))
|
||||
return ceiling / 2 + random.uniform(0, ceiling / 2)
|
||||
|
||||
|
||||
def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict:
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"job_id": str(job_id),
|
||||
"type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": fmt(created_at),
|
||||
}
|
||||
|
||||
|
||||
def parse_message(body: bytes) -> Optional[dict]:
|
||||
"""Strict check against jobs.schema.json; anything else goes to the DLQ untouched."""
|
||||
try:
|
||||
data = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
return None
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}:
|
||||
return None
|
||||
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
|
||||
return None
|
||||
if "traceparent" in data and not (isinstance(data["traceparent"], str)
|
||||
and telemetry.TRACEPARENT_RE.match(data["traceparent"])):
|
||||
return None
|
||||
if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512):
|
||||
return None
|
||||
for key in ("pipeline_version", "attempt"):
|
||||
if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1:
|
||||
return None
|
||||
try:
|
||||
data["job_id"] = str(uuid.UUID(data["job_id"]))
|
||||
data["attachment_id"] = str(uuid.UUID(data["attachment_id"]))
|
||||
except (ValueError, AttributeError, TypeError):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None):
|
||||
await conn.execute(
|
||||
"INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())",
|
||||
(job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})),
|
||||
)
|
||||
|
||||
|
||||
async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0):
|
||||
body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"])
|
||||
# W3C trace context of the transaction that schedules the job (upload request span or the failed consumer
|
||||
# span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate).
|
||||
body.update(telemetry.inject())
|
||||
await conn.execute(
|
||||
"INSERT INTO outbox (job_id, payload, available_at, created_at)"
|
||||
" VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())",
|
||||
(job["id"], json.dumps(body), delay),
|
||||
)
|
||||
|
||||
|
||||
async def enqueue(conn, attachment_id: str, actor: str) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts,"
|
||||
" next_attempt_at, created_at, updated_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())"
|
||||
" RETURNING id, attachment_id, pipeline_version, attempt, created_at",
|
||||
(attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
await _outbox(conn, job, 1)
|
||||
await audit(conn, job["id"], attachment_id, "enqueued", actor, 1)
|
||||
return job
|
||||
|
||||
|
||||
async def pending_count(conn) -> int:
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')")
|
||||
return (await cur.fetchone())["n"]
|
||||
|
||||
|
||||
async def queue_stats(conn) -> dict:
|
||||
"""Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows."""
|
||||
cur = await conn.execute(
|
||||
"SELECT count(*) FILTER (WHERE state = 'queued') AS queued,"
|
||||
" count(*) FILTER (WHERE state = 'processing') AS processing,"
|
||||
" coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age"
|
||||
" FROM attachment_jobs WHERE state IN ('queued', 'processing')"
|
||||
)
|
||||
row = dict(await cur.fetchone())
|
||||
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
|
||||
row["outbox"] = (await cur.fetchone())["n"]
|
||||
return row
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- worker side
|
||||
|
||||
JOB_COLUMNS = (
|
||||
"j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until,"
|
||||
" j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired,"
|
||||
" (j.next_attempt_at <= clock_timestamp()) AS due"
|
||||
)
|
||||
|
||||
|
||||
async def claim(conn, msg: dict, owner: str):
|
||||
"""Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version."""
|
||||
cur = await conn.execute(
|
||||
"SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status"
|
||||
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.id = %s FOR UPDATE OF j",
|
||||
(msg["job_id"],),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
if job is None or str(job["attachment_id"]) != msg["attachment_id"]:
|
||||
return "missing", None
|
||||
if job["state"] in ("succeeded", "failed"):
|
||||
return "done", job
|
||||
if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]:
|
||||
return "newer_version", job
|
||||
if msg["attempt"] != job["attempt"]:
|
||||
return "stale", job
|
||||
if job["state"] == "processing" and not job["lease_expired"]:
|
||||
return "busy", job
|
||||
if job["state"] == "queued" and not job["due"]:
|
||||
return "early", job
|
||||
takeover = job["state"] == "processing"
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'processing', lease_owner = %s,"
|
||||
" lease_until = clock_timestamp() + make_interval(secs => %s),"
|
||||
" started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp()),"
|
||||
" updated_at = clock_timestamp() WHERE id = %s",
|
||||
(owner, config.JOB_LEASE_SECONDS, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"])
|
||||
return "claimed", job
|
||||
|
||||
|
||||
async def heartbeat(conn, job_id, owner: str) -> bool:
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
|
||||
(config.JOB_LEASE_SECONDS, job_id, owner),
|
||||
)
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
async def attachment_source(conn, attachment_id) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status"
|
||||
" FROM attachments WHERE id = %s",
|
||||
(attachment_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool:
|
||||
"""Atomically publish the result. False when the lease was lost: someone else owns the effect now."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL,"
|
||||
" lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
|
||||
(job["id"], owner),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return False
|
||||
for v in variants:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,"
|
||||
" size_bytes, width, height, sha256, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())"
|
||||
" ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
|
||||
(job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
|
||||
v["size_bytes"], v["width"], v["height"], v["sha256"]),
|
||||
)
|
||||
# Never move a newer pipeline_version back, never touch an attachment that is already ready.
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s",
|
||||
(job["pipeline_version"], job["attachment_id"], job["pipeline_version"]),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"],
|
||||
{"variants": [v["name"] for v in variants]})
|
||||
return True
|
||||
|
||||
|
||||
async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str:
|
||||
attempt = job["attempt"]
|
||||
if transient and attempt < job["max_attempts"]:
|
||||
delay = backoff_seconds(attempt)
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL,"
|
||||
" next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s,"
|
||||
" updated_at = clock_timestamp() WHERE id = %s",
|
||||
(delay, detail[:500], error_code, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await _outbox(conn, job, attempt + 1, delay=delay)
|
||||
await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt,
|
||||
{"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)})
|
||||
return "retry"
|
||||
final_code = error_code if not transient else "retry_exhausted"
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(),"
|
||||
" last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(detail[:500], final_code, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status IN ('queued', 'processing')",
|
||||
(final_code, job["attachment_id"]),
|
||||
)
|
||||
await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt,
|
||||
{"error_code": final_code, "cause": error_code, "detail": detail[:200]})
|
||||
return "failed"
|
||||
|
||||
|
||||
async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str:
|
||||
"""Returns retry | failed | lost (lease is not ours any more: the new owner decides)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs"
|
||||
" WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE",
|
||||
(job["id"], owner),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "lost"
|
||||
return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient)
|
||||
|
||||
|
||||
async def sweep_leases(conn, actor: str) -> List[str]:
|
||||
"""Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner"
|
||||
" FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()"
|
||||
" ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED",
|
||||
)
|
||||
outcomes = []
|
||||
for row in await cur.fetchall():
|
||||
outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired",
|
||||
"lease of %s expired" % (row["lease_owner"] or "?"), True)
|
||||
outcomes.append("%s:%s" % (row["id"], outcome))
|
||||
return outcomes
|
||||
|
||||
|
||||
async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict:
|
||||
"""failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed."""
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at,"
|
||||
" a.status AS attachment_status, a.staging_key"
|
||||
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
|
||||
" WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a",
|
||||
(attachment_id,),
|
||||
)
|
||||
job = await cur.fetchone()
|
||||
if job is None:
|
||||
return {"result": "refused", "reason": "no job for attachment"}
|
||||
if job["state"] != "failed" or job["attachment_status"] != "failed":
|
||||
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"],
|
||||
{"state": job["state"], "attachment_status": job["attachment_status"]})
|
||||
return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])}
|
||||
if not job["staging_key"] or not await staging_exists(job["staging_key"]):
|
||||
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"})
|
||||
return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"}
|
||||
attempt = job["attempt"] + 1
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp(),"
|
||||
" finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s",
|
||||
(attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]),
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status = 'failed'",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
await _outbox(conn, job, attempt)
|
||||
await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt)
|
||||
return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt}
|
||||
|
||||
|
||||
async def failed_jobs(conn, limit: int = 50) -> List[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at"
|
||||
" FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s",
|
||||
(limit,),
|
||||
)
|
||||
return [
|
||||
{"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"],
|
||||
"max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"],
|
||||
"failed_at": fmt(r["updated_at"])}
|
||||
for r in await cur.fetchall()
|
||||
]
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def older_than(seconds: int) -> datetime:
|
||||
return utcnow() - timedelta(seconds=seconds)
|
||||
+801
@@ -0,0 +1,801 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import sys
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from app import config, faults, jobs, openbao, passwords, reconcile, repo, telemetry
|
||||
from app.cursor import BadCursor, decode_cursor, encode_cursor
|
||||
from app.db import Database
|
||||
from app.imagecheck import inspect_image
|
||||
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
|
||||
from app.broker import Publisher
|
||||
from app.outbox import Relay
|
||||
from app.s3 import S3Storage
|
||||
from app.sessions import SessionStore
|
||||
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
ZERO_WIDTH = str.maketrans({char: None for char in ""})
|
||||
EXEMPT_PATHS = ("/health/live", "/health/ready")
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
background = set()
|
||||
db = Database()
|
||||
sessions = SessionStore(db)
|
||||
s3 = S3Storage()
|
||||
publisher = Publisher()
|
||||
relay = Relay(db, publisher)
|
||||
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0,
|
||||
"queue": None}
|
||||
VARIANTS = ("original", "thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
def rss_mb() -> float:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
|
||||
except OSError:
|
||||
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
|
||||
|
||||
|
||||
def count_rejection(code: str) -> None:
|
||||
state["rejected"][code] = state["rejected"].get(code, 0) + 1
|
||||
|
||||
|
||||
async def sweeper() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
|
||||
try:
|
||||
removed = await sessions.sweep()
|
||||
if removed:
|
||||
log.info("sessions_expired removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("session_sweep_skipped: storage unavailable")
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||||
if removed:
|
||||
log.info("idempotency_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("idempotency_cleanup_skipped")
|
||||
try:
|
||||
removed = await relay.cleanup()
|
||||
if removed:
|
||||
log.info("outbox_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("outbox_cleanup_skipped")
|
||||
|
||||
|
||||
async def pending_watch() -> None:
|
||||
"""Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second."""
|
||||
while True:
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
queue = await jobs.queue_stats(conn)
|
||||
state["pending_jobs"] = queue["queued"] + queue["processing"]
|
||||
state["queue"] = queue
|
||||
except Exception:
|
||||
state["queue"] = None # unknown is not zero: the gauge disappears instead of lying
|
||||
await asyncio.sleep(config.PENDING_CHECK_SECONDS)
|
||||
|
||||
|
||||
async def reconciler_loop() -> None:
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
|
||||
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
|
||||
log.info("reconcile %s", result)
|
||||
except Exception:
|
||||
log.warning("reconcile_skipped: dependency unavailable")
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def readiness_watch() -> None:
|
||||
failures = 0
|
||||
while True:
|
||||
if await db.ready():
|
||||
failures = 0
|
||||
if not state["ready"]:
|
||||
state["ready"] = True
|
||||
log.info("ready_again: database reachable")
|
||||
else:
|
||||
failures += 1
|
||||
if failures >= config.READY_FAILURES and state["ready"]:
|
||||
state["ready"] = False
|
||||
log.warning("not_ready: database unreachable, business requests get 503")
|
||||
s3_ok = await run_in_threadpool(s3.probe)
|
||||
if s3_ok != state["s3_ok"]:
|
||||
state["s3_ok"] = s3_ok
|
||||
log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
|
||||
await asyncio.sleep(config.READY_CHECK_SECONDS)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
previous = {}
|
||||
|
||||
def on_signal(signum, frame):
|
||||
if not state["draining"]:
|
||||
state["draining"] = True
|
||||
log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name)
|
||||
handler = previous.get(signum)
|
||||
if callable(handler):
|
||||
handler(signum, frame)
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
previous[sig] = signal.getsignal(sig)
|
||||
signal.signal(sig, on_signal)
|
||||
await db.open()
|
||||
telemetry.register_pool(db)
|
||||
telemetry.register_queue(state)
|
||||
await sessions.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
state["s3_ok"] = await run_in_threadpool(s3.probe)
|
||||
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()),
|
||||
asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())]
|
||||
health_server = await asyncio.start_server(health_listener, "127.0.0.1", config.HEALTH_LISTEN_PORT)
|
||||
bao = openbao.client()
|
||||
if bao is not None:
|
||||
# Rotation without restart: a new KV version swaps DB and S3 credentials for new connections.
|
||||
bao.on_change(lambda data: db.rotate(data.get("database_url", "")))
|
||||
bao.on_change(s3.rotate)
|
||||
bao.start_refresher(config.BAO_REFRESH_SECONDS)
|
||||
log.info(
|
||||
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
|
||||
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
|
||||
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
|
||||
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
|
||||
)
|
||||
log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s",
|
||||
config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION)
|
||||
yield
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
health_server.close()
|
||||
await publisher.close()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def guard(request: Request, call_next):
|
||||
rid = uuid.uuid4().hex
|
||||
request.state.rid = rid
|
||||
# X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request.
|
||||
telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID)
|
||||
exempt = request.scope["path"] in EXEMPT_PATHS
|
||||
if state["draining"] and request.scope["path"] != "/health/live":
|
||||
count_rejection("draining")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
|
||||
if not exempt and not state["ready"]:
|
||||
count_rejection("not_ready")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
|
||||
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
|
||||
count_rejection("inflight")
|
||||
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
|
||||
state["inflight"] += 1
|
||||
try:
|
||||
fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else ""
|
||||
if fault:
|
||||
await inject_http_fault(fault)
|
||||
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
count_rejection("request_timeout")
|
||||
log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.scope["path"])
|
||||
response = error_body(
|
||||
rid, config.INSTANCE_ID, 503, "unavailable",
|
||||
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
|
||||
)
|
||||
except Exception as exc:
|
||||
telemetry.record_exception(exc)
|
||||
log.exception("unhandled_error request_id=%s", rid)
|
||||
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
finally:
|
||||
state["inflight"] -= 1
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
async def inject_http_fault(fault: str) -> None:
|
||||
"""Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error`
|
||||
raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose
|
||||
trace survival can be counted (span attribute messenger.fault)."""
|
||||
telemetry.set_attributes(messenger__fault=fault[:16])
|
||||
if fault == "slow":
|
||||
await asyncio.sleep(0.6)
|
||||
elif fault == "error":
|
||||
raise RuntimeError("injected handler failure")
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
if exc.status in (413, 429, 503):
|
||||
count_rejection(exc.code)
|
||||
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
def parse_uuid(value: object) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def non_blank(value: object, max_len: int) -> bool:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
|
||||
return False
|
||||
return value.translate(ZERO_WIDTH).strip() != ""
|
||||
|
||||
|
||||
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
|
||||
declared = request.headers.get("content-length")
|
||||
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
raw += chunk
|
||||
if len(raw) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
try:
|
||||
body = json.loads(bytes(raw).decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Некорректный JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
allowed = set(required) | set(optional)
|
||||
if set(body) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if set(required) - set(body):
|
||||
raise bad("Отсутствуют обязательные поля")
|
||||
return body
|
||||
|
||||
|
||||
def ensure_capacity() -> None:
|
||||
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
|
||||
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
token = value.strip()
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
async def actor(request: Request) -> dict:
|
||||
user = await sessions.resolve(bearer_token(request))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> Tuple[str, str]:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized("Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized("Basic") from None
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator:
|
||||
raise unauthorized("Basic")
|
||||
return username, password
|
||||
|
||||
|
||||
async def member_chat(user: dict, chat_id: str) -> dict:
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
chat = await repo.chat_for_member(conn, cid, user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
|
||||
def page_params(request: Request) -> Tuple[int, Optional[str]]:
|
||||
query = request.query_params
|
||||
limit = 50
|
||||
if "limit" in query:
|
||||
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
|
||||
raise bad("limit должен быть целым 1..100")
|
||||
limit = int(query["limit"])
|
||||
cursor = query.get("cursor")
|
||||
if cursor is not None and cursor == "":
|
||||
raise bad("Некорректный cursor")
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def keyset(scope: str, cursor: Optional[str]):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
return decode_cursor(scope, cursor)
|
||||
except BadCursor:
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
|
||||
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
|
||||
return JSONResponse({"items": items, "next_cursor": next_cursor})
|
||||
|
||||
|
||||
def detect_content_type(head: bytes) -> Optional[str]:
|
||||
if head[:8] == PNG_MAGIC:
|
||||
return "image/png"
|
||||
if head[:3] == JPEG_MAGIC:
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
|
||||
raise bad("filename: 1..255 символов, не только пробелы")
|
||||
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
|
||||
raise bad("filename содержит недопустимые символы")
|
||||
|
||||
|
||||
hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY)
|
||||
|
||||
|
||||
async def hash_slot(func, *args):
|
||||
"""Login DoS budget: at most PASSWORD_HASH_CONCURRENCY Argon2 computations per replica at a time (each ~19 MiB,
|
||||
measured time in REPORT.md); a request that cannot get a slot quickly gets 503 instead of queueing unbounded."""
|
||||
try:
|
||||
await asyncio.wait_for(hash_slots.acquire(), config.PASSWORD_HASH_QUEUE_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
count_rejection("hash_busy")
|
||||
raise ApiError(503, "unavailable", "Сервер перегружен проверкой паролей, повторите позже",
|
||||
{"Retry-After": "1"}) from None
|
||||
try:
|
||||
return await run_in_threadpool(func, *args)
|
||||
finally:
|
||||
hash_slots.release()
|
||||
|
||||
|
||||
def client_ip(request: Request) -> str:
|
||||
"""X-Real-IP is set by nginx. It can be trusted: only the LB identity passes the API mTLS check, so a client
|
||||
cannot reach the API directly and forge it. Fallback: the TCP peer."""
|
||||
value = request.headers.get("x-real-ip", "")
|
||||
return value if re.fullmatch(r"[0-9a-fA-F:.]{2,45}", value) else (request.client.host if request.client else "-")
|
||||
|
||||
|
||||
def rate_limited(scope: str, retry_after: int) -> None:
|
||||
count_rejection("login_rate_" + scope)
|
||||
log.info("login_rate_limited scope=%s retry_after=%d", scope, retry_after)
|
||||
raise ApiError(429, "rate_limited", "Слишком много попыток входа, повторите позже",
|
||||
{"Retry-After": str(int(retry_after))})
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
async def health_listener(reader, writer) -> None:
|
||||
"""Loopback-only (127.0.0.1) plain HTTP for the Docker healthcheck: /health/live and /health/ready only.
|
||||
It is not published and not reachable from the network, so it is no bypass of the mTLS business listener."""
|
||||
try:
|
||||
line = (await asyncio.wait_for(reader.readline(), 2)).decode("latin-1")
|
||||
path = line.split(" ")[1] if line.count(" ") >= 2 else ""
|
||||
if path == "/health/live":
|
||||
status = 200
|
||||
elif path == "/health/ready":
|
||||
status = 200 if (not state["draining"] and state["s3_ok"] and await db.ready()) else 503
|
||||
else:
|
||||
status = 404
|
||||
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
|
||||
writer.write(b"HTTP/1.1 %d X\r\nContent-Type: application/json\r\nContent-Length: %d\r\n"
|
||||
b"Connection: close\r\n\r\n%s" % (status, len(body), body))
|
||||
await writer.drain()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not await db.ready():
|
||||
raise unavailable("База данных недоступна")
|
||||
if not state["s3_ok"]:
|
||||
raise unavailable("Хранилище файлов недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
body = await read_json(request, ("username", "display_name", "password"))
|
||||
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
if not non_blank(body["display_name"], 100):
|
||||
raise bad("display_name: 1..100 символов, не только пробелы")
|
||||
password = body["password"]
|
||||
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
|
||||
ensure_capacity()
|
||||
password_hash = await hash_slot(passwords.hash_password, password)
|
||||
async with db.connection() as conn:
|
||||
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
|
||||
if user is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
log.info("user_created id=%s", user["id"])
|
||||
return JSONResponse(user, status_code=201)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def login(request: Request):
|
||||
if await request.body():
|
||||
raise bad("POST /auth/sessions не принимает тело запроса")
|
||||
username, password = basic_credentials(request)
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
async with db.connection() as conn:
|
||||
row = await repo.credentials(conn, username)
|
||||
# lab7: limits BEFORE the expensive Argon2 verification; counters are shared by all replicas (Valkey).
|
||||
ip_key = "ratelimit:login:ip:" + client_ip(request)
|
||||
user_key = "ratelimit:login:user:" + (username if USERNAME_RE.fullmatch(username) else "-invalid-")
|
||||
count, retry = await sessions.hit(ip_key, config.LOGIN_RATE_WINDOW_SECONDS)
|
||||
if count > config.LOGIN_RATE_LIMIT_PER_IP:
|
||||
rate_limited("ip", retry)
|
||||
failures, retry = await sessions.peek(user_key)
|
||||
if failures >= config.LOGIN_RATE_LIMIT_PER_USER:
|
||||
rate_limited("user", retry)
|
||||
stored = row["password_hash"] if row else passwords.DUMMY_HASH
|
||||
matches = await hash_slot(passwords.verify_password, stored, password)
|
||||
if row is None or not matches:
|
||||
await sessions.hit(user_key, config.LOGIN_RATE_WINDOW_SECONDS)
|
||||
log.info("login_failed username_known=%s", row is not None)
|
||||
raise unauthorized("Basic")
|
||||
await sessions.reset(user_key)
|
||||
if passwords.needs_rehash(stored):
|
||||
# Old parameters (or another algorithm) -> re-hash with the current Argon2id parameters while we know the
|
||||
# plaintext; a concurrent login doing the same is harmless (both hashes are valid).
|
||||
new_hash = await hash_slot(passwords.hash_password, password)
|
||||
async with db.connection() as conn:
|
||||
await repo.update_password_hash(conn, row["id"], new_hash)
|
||||
log.info("password_rehashed user=%s", row["id"])
|
||||
user = repo.public_user(row)
|
||||
token, expires_at = await sessions.create(user)
|
||||
log.info("session_created user=%s", user["id"])
|
||||
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def logout(request: Request):
|
||||
token = bearer_token(request)
|
||||
user = await sessions.resolve(token)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
await sessions.revoke(token)
|
||||
log.info("session_revoked user=%s", user["id"])
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def me(request: Request):
|
||||
return await actor(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
body = await read_json(request, ("title", "member_ids"))
|
||||
if not non_blank(body["title"], 100):
|
||||
raise bad("title: 1..100 символов, не только пробелы")
|
||||
raw_ids = body["member_ids"]
|
||||
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
|
||||
raise bad("member_ids: массив из 1..100 UUID")
|
||||
ids: List[str] = []
|
||||
for value in raw_ids:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("member_ids содержит некорректный UUID")
|
||||
ids.append(parsed)
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user["id"] not in ids:
|
||||
raise bad("Создатель должен входить в member_ids")
|
||||
ensure_capacity()
|
||||
|
||||
async def insert(conn):
|
||||
return await repo.create_chat(conn, body["title"], ids)
|
||||
|
||||
try:
|
||||
chat = await db.in_transaction(insert)
|
||||
except repo.UnknownMember:
|
||||
raise bad("member_ids содержит неизвестного пользователя") from None
|
||||
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
|
||||
return JSONResponse(chat, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "chats:" + user["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
return await member_chat(await actor(request), chat_id)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
|
||||
idem_key = request.headers.get("idempotency-key", "")
|
||||
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
|
||||
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
body = await read_json(request, ("text",), ("attachment_ids",))
|
||||
|
||||
text = body["text"]
|
||||
attachment_ids_raw = body.get("attachment_ids", [])
|
||||
|
||||
if not isinstance(attachment_ids_raw, list):
|
||||
raise bad("attachment_ids должен быть массивом UUID")
|
||||
if len(attachment_ids_raw) > 10:
|
||||
raise bad("attachment_ids: максимум 10 вложений")
|
||||
|
||||
attachment_ids: List[str] = []
|
||||
for value in attachment_ids_raw:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("attachment_ids содержит некорректный UUID")
|
||||
attachment_ids.append(parsed)
|
||||
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
|
||||
has_text = isinstance(text, str) and non_blank(text, 4000)
|
||||
has_attachments = len(attachment_ids) > 0
|
||||
|
||||
if not has_text and not has_attachments:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if isinstance(text, str) and len(text) > 4000:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise bad("text должен быть строкой")
|
||||
|
||||
ensure_capacity()
|
||||
|
||||
normalized = {"text": text}
|
||||
if attachment_ids:
|
||||
normalized["attachment_ids"] = attachment_ids
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
async def do(conn):
|
||||
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
|
||||
|
||||
status, message = await db.in_transaction(do)
|
||||
|
||||
if status == "conflict":
|
||||
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
|
||||
if status == "not_member":
|
||||
raise not_found()
|
||||
if status == "attachment_not_found":
|
||||
raise not_found()
|
||||
if status == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
|
||||
return JSONResponse(message, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def list_messages(chat_id: str, request: Request):
|
||||
chat = await member_chat(await actor(request), chat_id)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "messages:" + chat["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
chat = await member_chat(user, chat_id)
|
||||
|
||||
filename = request.query_params.get("filename", "")
|
||||
validate_filename(filename)
|
||||
|
||||
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if declared_ct not in ALLOWED_CONTENT_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
|
||||
|
||||
declared_length = request.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
|
||||
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
ensure_capacity()
|
||||
# Backpressure: do not accept more work than the pipeline can hold; the client retries later.
|
||||
if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS:
|
||||
count_rejection("queue_full")
|
||||
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
|
||||
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
|
||||
head = b""
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
if len(head) < 8:
|
||||
head += chunk[: 8 - len(head)]
|
||||
if len(head) == 8 and detect_content_type(head) is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
sha.update(chunk)
|
||||
spool.write(chunk)
|
||||
if size == 0:
|
||||
raise bad("Пустой файл")
|
||||
actual_ct = detect_content_type(head)
|
||||
if actual_ct is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
if actual_ct != declared_ct:
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(inspect_image, spool, size, actual_ct)
|
||||
|
||||
attachment_id = str(uuid.uuid4())
|
||||
staging_key = "staging/" + attachment_id
|
||||
object_key = "attachments/%s/original" % attachment_id
|
||||
async with db.connection() as conn:
|
||||
await repo.begin_attachment(
|
||||
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(),
|
||||
staging_key, object_key,
|
||||
)
|
||||
try:
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct)
|
||||
except BaseException:
|
||||
defer(compensate(attachment_id, staging_key))
|
||||
raise
|
||||
|
||||
# Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox.
|
||||
try:
|
||||
att = await db.in_transaction(lambda conn: repo.queue_attachment(
|
||||
conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS))
|
||||
except BaseException:
|
||||
await compensate(attachment_id, staging_key)
|
||||
raise
|
||||
if att == repo.QUEUE_FULL:
|
||||
defer(compensate(attachment_id, staging_key))
|
||||
count_rejection("queue_full")
|
||||
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
|
||||
if att is None:
|
||||
raise unavailable("Загрузка отменена сверкой, повторите запрос")
|
||||
telemetry.set_attributes(messenger__attachment_id=att["id"])
|
||||
if faults.active(filename, "api_crash_after_commit"):
|
||||
faults.crash("API after COMMIT of job/outbox, before publish")
|
||||
relay.notify()
|
||||
log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size)
|
||||
return JSONResponse(att, status_code=202)
|
||||
|
||||
|
||||
def defer(coro) -> None:
|
||||
task = asyncio.ensure_future(coro)
|
||||
background.add(task)
|
||||
task.add_done_callback(background.discard)
|
||||
|
||||
|
||||
async def compensate(attachment_id: str, object_key: str) -> None:
|
||||
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
await repo.abort_attachment(conn, attachment_id)
|
||||
await run_in_threadpool(s3.delete, object_key)
|
||||
except BaseException:
|
||||
log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id)
|
||||
|
||||
|
||||
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
|
||||
user = await actor(request)
|
||||
aid = parse_uuid(attachment_id)
|
||||
if aid is None:
|
||||
raise bad("attachment_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
row = await repo.get_attachment(conn, aid)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def get_attachment(attachment_id: str, request: Request):
|
||||
row = await accessible_attachment(request, attachment_id)
|
||||
variants = []
|
||||
if row["status"] == "ready":
|
||||
async with db.connection() as conn:
|
||||
variants = await repo.get_variants(conn, row["id"], row["pipeline_version"])
|
||||
return repo._attachment(row, variants)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(attachment_id: str, request: Request):
|
||||
row = await accessible_attachment(request, attachment_id)
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant not in VARIANTS:
|
||||
raise bad("variant: original, thumbnail, crop или optimized")
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing")
|
||||
else "Обработка вложения завершилась ошибкой")
|
||||
key, content_type = row["object_key"], row["content_type"]
|
||||
if variant != "original":
|
||||
async with db.connection() as conn:
|
||||
found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant]
|
||||
if not found:
|
||||
raise not_found()
|
||||
key, content_type = found[0]["object_key"], found[0]["content_type"]
|
||||
opened = await run_in_threadpool(s3.open_stream, key)
|
||||
if opened is None:
|
||||
log.error("object_missing attachment=%s variant=%s", row["id"], variant)
|
||||
raise unavailable("Файл временно недоступен")
|
||||
chunks, length = opened
|
||||
return StreamingResponse(
|
||||
iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)},
|
||||
)
|
||||
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def fallback(request: Request):
|
||||
raise not_found()
|
||||
|
||||
|
||||
# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel
|
||||
# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics.
|
||||
telemetry.setup("messenger-api", app)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""OpenBao client for the services (lab7): cert auth -> short-lived token -> read ONE scoped KV v2 secret.
|
||||
|
||||
Bootstrap trust: the only material a service gets from the operator is its TLS client certificate (Docker secret).
|
||||
It is also its identity for RabbitMQ (EXTERNAL) and LB->API mTLS is separate. There is no "secret to get secrets"
|
||||
in the image or in .env: no token, no AppRole SecretID.
|
||||
|
||||
login POST /v1/auth/cert/login (client certificate, role = BAO_AUTH_ROLE) -> token (TTL 15 min, max 1 h)
|
||||
read GET /v1/messenger/data/<BAO_KV_PATH> (policy messenger-<role>: read own path only)
|
||||
refresh every BAO_REFRESH_SECONDS: renew-self (or re-login near max TTL), re-read metadata; a new KV version
|
||||
(rotation) is delivered to registered callbacks without a restart
|
||||
sealed / unreachable OpenBao: the process keeps the secrets it already holds (bounded: existing DB/S3/broker
|
||||
sessions continue); a NEW process waits up to BAO_STARTUP_TIMEOUT_SECONDS and then exits non-zero.
|
||||
Secret values are never logged; only paths, versions, TTLs and error classes.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
log = logging.getLogger("messenger.openbao")
|
||||
|
||||
|
||||
class OpenBaoError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class OpenBao:
|
||||
def __init__(self) -> None:
|
||||
self.addr = os.environ["BAO_ADDR"].rstrip("/")
|
||||
self.role = os.environ["BAO_AUTH_ROLE"]
|
||||
self.path = os.environ["BAO_KV_PATH"]
|
||||
self.mount = os.environ.get("BAO_KV_MOUNT", "messenger")
|
||||
self.ctx = ssl.create_default_context(cafile=os.environ.get("TLS_CA_FILE", "/run/secrets/ca.crt"))
|
||||
self.ctx.load_cert_chain(os.environ.get("TLS_CLIENT_CERT_FILE", "/run/secrets/client.crt"),
|
||||
os.environ.get("TLS_CLIENT_KEY_FILE", "/run/secrets/client.key"))
|
||||
self.token: Optional[str] = None
|
||||
self.token_expires = 0.0
|
||||
self.data: Dict[str, str] = {}
|
||||
self.version = 0
|
||||
self.callbacks: List[Callable[[Dict[str, str]], None]] = []
|
||||
self.state = {"available": False, "last_ok": 0.0, "errors": 0, "logins": 0, "renewals": 0}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ HTTP
|
||||
def _call(self, method: str, path: str, body: Optional[dict] = None, token: Optional[str] = None) -> dict:
|
||||
req = urllib.request.Request(self.addr + path, method=method,
|
||||
data=json.dumps(body).encode() if body is not None else None)
|
||||
req.add_header("Content-Type", "application/json")
|
||||
if token:
|
||||
req.add_header("X-Vault-Token", token)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=self.ctx, timeout=5) as response:
|
||||
raw = response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
# 503 = sealed / standby, 403 = denied or expired token, 400 = cert not accepted for the role
|
||||
raise OpenBaoError("HTTP %d on %s" % (exc.code, path.split("?")[0])) from None
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
raise OpenBaoError("%s on %s" % (type(getattr(exc, "reason", exc)).__name__, path)) from None
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
def login(self) -> None:
|
||||
auth = self._call("POST", "/v1/auth/cert/login", {"name": self.role})["auth"]
|
||||
now = time.time()
|
||||
self.token = auth["client_token"]
|
||||
self.token_expires = now + auth["lease_duration"]
|
||||
self.state["logins"] += 1
|
||||
log.info("openbao_login role=%s policies=%s ttl=%ds", self.role, ",".join(auth.get("policies", [])),
|
||||
auth["lease_duration"])
|
||||
|
||||
def _ensure_token(self, interval: float) -> None:
|
||||
now = time.time()
|
||||
if self.token is None or now > self.token_expires - 1:
|
||||
self.login()
|
||||
return
|
||||
if self.token_expires - now < 2 * interval + 5:
|
||||
try:
|
||||
auth = self._call("POST", "/v1/auth/token/renew-self", {}, self.token)["auth"]
|
||||
self.token_expires = time.time() + auth["lease_duration"]
|
||||
self.state["renewals"] += 1
|
||||
if auth["lease_duration"] < 2 * interval + 5: # reached max TTL: authenticate again
|
||||
self.login()
|
||||
except OpenBaoError:
|
||||
self.login()
|
||||
|
||||
def _read(self) -> None:
|
||||
body = self._call("GET", "/v1/%s/data/%s" % (self.mount, self.path), token=self.token)["data"]
|
||||
version = int(body["metadata"]["version"])
|
||||
changed = version != self.version
|
||||
with self.lock:
|
||||
self.data, self.version = dict(body["data"]), version
|
||||
self.state.update(available=True, last_ok=time.time())
|
||||
if changed:
|
||||
log.info("openbao_secret_loaded path=%s/%s version=%d keys=%s", self.mount, self.path, version,
|
||||
",".join(sorted(self.data)))
|
||||
for callback in list(self.callbacks):
|
||||
try:
|
||||
callback(dict(self.data))
|
||||
except Exception:
|
||||
log.exception("openbao_rotation_callback_failed")
|
||||
|
||||
# ------------------------------------------------------------------ lifecycle
|
||||
def load_blocking(self, timeout: float) -> Dict[str, str]:
|
||||
deadline = time.monotonic() + timeout
|
||||
delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
self.login()
|
||||
self._read()
|
||||
return dict(self.data)
|
||||
except (OpenBaoError, KeyError, ValueError) as exc:
|
||||
self.state["errors"] += 1
|
||||
if time.monotonic() > deadline:
|
||||
log.error("openbao_unavailable_at_start role=%s error=%s: giving up", self.role, exc)
|
||||
raise SystemExit(78)
|
||||
log.warning("openbao_not_ready role=%s error=%s (sealed or down?), retry in %.0fs", self.role, exc, delay)
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 2, 10.0)
|
||||
|
||||
def on_change(self, callback: Callable[[Dict[str, str]], None]) -> None:
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def start_refresher(self, interval: float) -> None:
|
||||
def loop():
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
self._ensure_token(interval)
|
||||
meta = self._call("GET", "/v1/%s/metadata/%s" % (self.mount, self.path), token=self.token)
|
||||
if int(meta["data"]["current_version"]) != self.version:
|
||||
self._read()
|
||||
else:
|
||||
self.state.update(available=True, last_ok=time.time())
|
||||
except (OpenBaoError, KeyError, ValueError) as exc:
|
||||
self.state["errors"] += 1
|
||||
if self.state["available"]:
|
||||
log.warning("openbao_unavailable role=%s error=%s: keeping cached secret version %d",
|
||||
self.role, exc, self.version)
|
||||
self.state["available"] = False
|
||||
if "403" in str(exc):
|
||||
self.token = None # expired/revoked token: log in again next round
|
||||
|
||||
threading.Thread(target=loop, name="openbao-refresh", daemon=True).start()
|
||||
|
||||
|
||||
_client: Optional[OpenBao] = None
|
||||
|
||||
|
||||
def client() -> Optional[OpenBao]:
|
||||
return _client
|
||||
|
||||
|
||||
def load(timeout: float) -> Dict[str, str]:
|
||||
"""Called once by app.config at import time when BAO_ADDR is set."""
|
||||
global _client
|
||||
_client = OpenBao()
|
||||
return _client.load_blocking(timeout)
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work).
|
||||
|
||||
Rows are locked while they are published; `published_at` is set in the same transaction only after the
|
||||
broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are
|
||||
expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish
|
||||
leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import config, telemetry
|
||||
|
||||
log = logging.getLogger("messenger.outbox")
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, db, publisher) -> None:
|
||||
self.db = db
|
||||
self.publisher = publisher
|
||||
self.wake = asyncio.Event()
|
||||
self.broker_ok = False
|
||||
self.published = 0
|
||||
|
||||
def notify(self) -> None:
|
||||
self.wake.set()
|
||||
|
||||
async def run_once(self) -> int:
|
||||
sent, failure = 0, None
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
cur = await conn.execute(
|
||||
"SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()"
|
||||
" ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED",
|
||||
(config.OUTBOX_BATCH,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
try:
|
||||
await self.publish(row["payload"])
|
||||
except Exception as exc:
|
||||
# Keep what was already confirmed: commit those rows, record the error, stop the batch.
|
||||
await conn.execute(
|
||||
"UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s",
|
||||
(type(exc).__name__, row["id"]),
|
||||
)
|
||||
failure = exc
|
||||
break
|
||||
await conn.execute(
|
||||
"UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1"
|
||||
" WHERE id = %s",
|
||||
(row["id"],),
|
||||
)
|
||||
sent += 1
|
||||
self.published += sent
|
||||
if failure is not None:
|
||||
raise failure
|
||||
return sent
|
||||
|
||||
async def publish(self, payload: dict) -> None:
|
||||
"""PRODUCER span, child of the context stored with the job (upload request or failed attempt);
|
||||
the message then carries the producer's own traceparent, so the consumer becomes its child."""
|
||||
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish",
|
||||
"messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"],
|
||||
"messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]}
|
||||
with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER,
|
||||
context=telemetry.extract(payload), attributes=attrs):
|
||||
body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")}
|
||||
body.update(telemetry.inject())
|
||||
await self.publisher.publish(body)
|
||||
|
||||
async def loop(self) -> None:
|
||||
delay = config.OUTBOX_POLL_SECONDS
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait_for(self.wake.wait(), delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
self.wake.clear()
|
||||
try:
|
||||
while await self.run_once() == config.OUTBOX_BATCH:
|
||||
pass
|
||||
if not self.broker_ok:
|
||||
log.info("outbox_relay_resumed: broker reachable")
|
||||
self.broker_ok = True
|
||||
delay = config.OUTBOX_POLL_SECONDS
|
||||
except Exception as exc:
|
||||
if self.broker_ok:
|
||||
log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__)
|
||||
self.broker_ok = False
|
||||
await self.publisher.close()
|
||||
delay = min(10.0, max(delay * 2, 1.0))
|
||||
|
||||
async def cleanup(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)",
|
||||
(config.OUTBOX_RETENTION_SECONDS,),
|
||||
)
|
||||
return cur.rowcount
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Argon2id password storage (lab7). Parameters come from the environment and were measured on the stand
|
||||
(student-tests/argon2_bench.py, REPORT.md); the PHC string stores salt + parameters, so old hashes stay valid and
|
||||
are upgraded on the next successful login (needs_rehash)."""
|
||||
from argon2 import PasswordHasher, Type
|
||||
from argon2.exceptions import Argon2Error, InvalidHashError, VerifyMismatchError
|
||||
|
||||
from app import config
|
||||
|
||||
HASHER = PasswordHasher(time_cost=config.ARGON2_ITERATIONS, memory_cost=config.ARGON2_MEMORY_KIB,
|
||||
parallelism=config.ARGON2_PARALLELISM, hash_len=32, salt_len=16, type=Type.ID)
|
||||
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return HASHER.hash(password)
|
||||
|
||||
|
||||
def verify_password(stored: str, password: str) -> bool:
|
||||
try:
|
||||
return HASHER.verify(stored, password)
|
||||
except (VerifyMismatchError, Argon2Error, InvalidHashError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def needs_rehash(stored: str) -> bool:
|
||||
try:
|
||||
return HASHER.check_needs_rehash(stored)
|
||||
except (InvalidHashError, ValueError):
|
||||
return True
|
||||
@@ -0,0 +1,155 @@
|
||||
"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout).
|
||||
|
||||
Key layout (all keys are generated by the server):
|
||||
staging/<attachment_id> written by the API before 202, read by the worker
|
||||
attachments/<attachment_id>/original server-side copy made by the worker
|
||||
attachments/<attachment_id>/v<N>/<variant>.* variants of pipeline version N
|
||||
Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row.
|
||||
|
||||
Cases (all gated by a grace period so in-flight work is never touched):
|
||||
stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both
|
||||
orphan_object object without any row/variant row, older than grace -> delete object
|
||||
staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete
|
||||
staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete
|
||||
(after that a replay is refused; the attachment stays failed and visible)
|
||||
missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing
|
||||
Queued/processing work is never touched: its staging object is the input of a live job.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from app import config
|
||||
|
||||
log = logging.getLogger("messenger.reconcile")
|
||||
|
||||
LOCK_ID = 2026_0104
|
||||
PREFIXES = ("staging/", "attachments/")
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
|
||||
started = time.monotonic()
|
||||
result = {
|
||||
"dry_run": dry_run, "grace_seconds": grace_seconds,
|
||||
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0,
|
||||
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
|
||||
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
|
||||
}
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
|
||||
if not (await cur.fetchone())["got"]:
|
||||
return {"skipped": "another reconciler is running"}
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=grace_seconds)
|
||||
await _stale_uploading(conn, s3, cutoff, dry_run, result)
|
||||
await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result)
|
||||
await _orphan_objects(conn, s3, cutoff, dry_run, result)
|
||||
await _missing_objects(conn, s3, cutoff, dry_run, result)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
result["seconds"] = round(time.monotonic() - started, 3)
|
||||
return result
|
||||
|
||||
|
||||
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, coalesce(staging_key, object_key) AS key FROM attachments"
|
||||
" WHERE status = 'uploading' AND created_at < %s",
|
||||
(cutoff,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
modified = await run_in_threadpool(s3.head, row["key"])
|
||||
if modified is not None and modified >= cutoff:
|
||||
result["protected_fresh_objects"] += 1
|
||||
continue
|
||||
result["stale_uploading"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
if modified is not None:
|
||||
await run_in_threadpool(s3.delete, row["key"])
|
||||
result["deleted_objects"] += 1
|
||||
done = await conn.execute(
|
||||
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
|
||||
)
|
||||
result["removed_rows"] += done.rowcount
|
||||
|
||||
|
||||
async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL"
|
||||
" AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
|
||||
(retention_cutoff,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
if await run_in_threadpool(s3.head, row["staging_key"]) is None:
|
||||
continue
|
||||
result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
again = await conn.execute(
|
||||
"SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
|
||||
(row["id"], retention_cutoff),
|
||||
)
|
||||
if await again.fetchone() is not None:
|
||||
await run_in_threadpool(s3.delete, row["staging_key"])
|
||||
result["deleted_objects"] += 1
|
||||
|
||||
|
||||
async def _known_keys(conn, keys):
|
||||
cur = await conn.execute(
|
||||
"SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)"
|
||||
" UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)"
|
||||
" UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)",
|
||||
(keys, keys, keys),
|
||||
)
|
||||
return {row["k"] for row in await cur.fetchall()}
|
||||
|
||||
|
||||
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
|
||||
for prefix in PREFIXES:
|
||||
pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix)))
|
||||
while True:
|
||||
page = await run_in_threadpool(next, pages, None)
|
||||
if page is None:
|
||||
break
|
||||
result["objects_scanned"] += len(page)
|
||||
keys = [o["Key"] for o in page]
|
||||
if not keys:
|
||||
continue
|
||||
known = await _known_keys(conn, keys)
|
||||
for obj in page:
|
||||
if obj["Key"] in known or obj["LastModified"] >= cutoff:
|
||||
continue
|
||||
result["orphan_objects"] += 1
|
||||
if not dry_run and not await _known_keys(conn, [obj["Key"]]):
|
||||
await run_in_threadpool(s3.delete, obj["Key"])
|
||||
result["deleted_objects"] += 1
|
||||
|
||||
|
||||
async def _missing_objects(conn, s3, cutoff, dry_run, result):
|
||||
last = None
|
||||
while True:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
|
||||
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
|
||||
(cutoff, last, last, BATCH),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
result["rows_scanned"] += 1
|
||||
if await run_in_threadpool(s3.exists, row["object_key"]):
|
||||
continue
|
||||
result["missing_objects"] += 1
|
||||
if not dry_run:
|
||||
done = await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
|
||||
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
|
||||
)
|
||||
result["marked_failed"] += done.rowcount
|
||||
last = str(rows[-1]["id"])
|
||||
+342
@@ -0,0 +1,342 @@
|
||||
import hashlib
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import psycopg
|
||||
|
||||
from app import config, jobs
|
||||
from app.cursor import fmt
|
||||
|
||||
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
|
||||
CHAT_QUERY = (
|
||||
"SELECT c.id, c.title, c.created_at,"
|
||||
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
|
||||
)
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def decoded(value) -> str:
|
||||
return bytes(value).decode("utf-8")
|
||||
|
||||
|
||||
def public_user(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": decoded(row["display_name"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _chat(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": decoded(row["title"]),
|
||||
"member_ids": [str(value) for value in row["member_ids"]],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": decoded(row["text"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
|
||||
msg = _message(row)
|
||||
msg["attachment_ids"] = attachment_ids
|
||||
return msg
|
||||
|
||||
|
||||
VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2}
|
||||
|
||||
|
||||
def _variant(row) -> dict:
|
||||
return {
|
||||
"name": row["name"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"width": row["width"],
|
||||
"height": row["height"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
|
||||
|
||||
def _attachment(row, variants=None) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
"updated_at": fmt(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
|
||||
try:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
|
||||
" RETURNING id, username, display_name, created_at",
|
||||
(username, display_name.encode("utf-8"), password_hash),
|
||||
)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
return None
|
||||
return public_user(await cur.fetchone())
|
||||
|
||||
|
||||
async def update_password_hash(conn, user_id, password_hash: str) -> None:
|
||||
await conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (password_hash, user_id))
|
||||
|
||||
|
||||
async def credentials(conn, username: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
|
||||
" RETURNING id, title, created_at",
|
||||
(title.encode("utf-8"),),
|
||||
)
|
||||
chat = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
|
||||
(chat["id"], member_ids),
|
||||
)
|
||||
if cur.rowcount != len(member_ids):
|
||||
raise UnknownMember()
|
||||
chat["member_ids"] = list(member_ids)
|
||||
return _chat(chat)
|
||||
|
||||
|
||||
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _chat(row) if row else None
|
||||
|
||||
|
||||
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY
|
||||
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY c.created_at, c.id LIMIT %s",
|
||||
(user_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_chat(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
|
||||
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _message(row) if row else None
|
||||
|
||||
|
||||
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
|
||||
cur = await conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
||||
(message_id,),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
|
||||
for pos, aid in enumerate(attachment_ids):
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
(message_id, aid, pos),
|
||||
)
|
||||
|
||||
|
||||
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
|
||||
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
messages = []
|
||||
for row in rows[:limit]:
|
||||
aids = await _fetch_message_attachment_ids(conn, row["id"])
|
||||
messages.append(_message_with_attachments(row, aids))
|
||||
return messages, len(rows) > limit
|
||||
|
||||
|
||||
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
|
||||
raw = "%s:%s:%s" % (user_id, chat_id, key)
|
||||
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
|
||||
|
||||
|
||||
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
|
||||
attachment_ids: List[str], idem_key: str, body_hash: str):
|
||||
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
)
|
||||
if await cur.fetchone() is None:
|
||||
return "not_member", None
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT request_body_hash, response_json FROM idempotency_keys"
|
||||
" WHERE user_id = %s AND chat_id = %s AND key = %s",
|
||||
(sender_id, chat_id, idem_key),
|
||||
)
|
||||
existing = await cur.fetchone()
|
||||
|
||||
if existing:
|
||||
if existing["request_body_hash"] != body_hash:
|
||||
return "conflict", None
|
||||
return "replay", json.loads(existing["response_json"])
|
||||
|
||||
if attachment_ids:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
|
||||
(attachment_ids,),
|
||||
)
|
||||
att_rows = await cur.fetchall()
|
||||
found_ids = {str(row["id"]) for row in att_rows}
|
||||
for aid in attachment_ids:
|
||||
if aid not in found_ids:
|
||||
return "attachment_not_found", None
|
||||
for row in att_rows:
|
||||
if str(row["chat_id"]) != chat_id:
|
||||
return "attachment_not_found", None
|
||||
if row["status"] == "failed":
|
||||
return "attachment_failed", None
|
||||
|
||||
text_bytes = text.encode("utf-8") if text else b""
|
||||
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text_bytes, chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "not_member", None
|
||||
|
||||
if attachment_ids:
|
||||
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
|
||||
|
||||
message = _message_with_attachments(row, list(attachment_ids))
|
||||
|
||||
await conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
|
||||
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
return "created", message
|
||||
|
||||
|
||||
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
|
||||
(ttl_seconds,),
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
|
||||
content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
||||
" staging_key, status, error_code, pipeline_version, created_at, updated_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key,
|
||||
config.PIPELINE_VERSION),
|
||||
)
|
||||
|
||||
|
||||
ATTACHMENT_COLUMNS = (
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status,"
|
||||
" error_code, pipeline_version, created_at, updated_at"
|
||||
)
|
||||
|
||||
|
||||
QUEUE_FULL = "queue_full"
|
||||
ADMISSION_LOCK = 2026_0105
|
||||
|
||||
|
||||
async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0):
|
||||
"""uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff).
|
||||
|
||||
With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock,
|
||||
so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight.
|
||||
"""
|
||||
if max_pending:
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,))
|
||||
if await jobs.pending_count(conn) >= max_pending:
|
||||
return QUEUE_FULL
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS,
|
||||
(attachment_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
await jobs.enqueue(conn, attachment_id, actor)
|
||||
return _attachment(row)
|
||||
|
||||
|
||||
async def abort_attachment(conn, attachment_id: str) -> None:
|
||||
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
async def get_attachment(conn, attachment_id: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'",
|
||||
(attachment_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]:
|
||||
cur = await conn.execute(
|
||||
"SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants"
|
||||
" WHERE attachment_id = %s AND pipeline_version = %s",
|
||||
(attachment_id, pipeline_version),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99))
|
||||
@@ -0,0 +1,132 @@
|
||||
import logging
|
||||
from typing import Iterator, Optional, Tuple
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import config
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.s3")
|
||||
|
||||
MISSING = ("NoSuchKey", "404", "NotFound")
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(self) -> None:
|
||||
self.client = None
|
||||
self.probe_client = None
|
||||
|
||||
def open(self, access_key: str = "", secret_key: str = "") -> None:
|
||||
self.access_key = access_key or config.S3_ACCESS_KEY_ID
|
||||
self.secret_key = secret_key or config.S3_SECRET_ACCESS_KEY
|
||||
self.probe_client = self._client(1.0, 1.5)
|
||||
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
|
||||
|
||||
def rotate(self, secrets: dict) -> None:
|
||||
"""New S3 credentials from OpenBao: swap the clients (boto3 clients are immutable)."""
|
||||
key, secret = secrets.get("s3_access_key_id"), secrets.get("s3_secret_access_key")
|
||||
if key and secret and (key, secret) != (self.access_key, self.secret_key):
|
||||
self.open(key, secret)
|
||||
log.info("s3_credentials_rotated access_key=%s", key)
|
||||
|
||||
def _client(self, connect: float, read: float):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=self.access_key,
|
||||
aws_secret_access_key=self.secret_key,
|
||||
verify=config.TLS_CA_FILE, # lab7: HTTPS to MinIO, server certificate verified against the lab CA
|
||||
config=BotoConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=connect,
|
||||
read_timeout=read,
|
||||
retries={"max_attempts": 0},
|
||||
),
|
||||
)
|
||||
|
||||
def probe(self) -> bool:
|
||||
try:
|
||||
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
self.client.put_object(
|
||||
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
|
||||
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
except BotoCoreError as exc:
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
body = response["Body"]
|
||||
return body.iter_chunks(65536), int(response["ContentLength"])
|
||||
|
||||
def head(self, key: str):
|
||||
"""LastModified of the object, or None when it does not exist."""
|
||||
try:
|
||||
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
raise
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
def list_pages(self, prefix: str):
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
|
||||
yield page.get("Contents", [])
|
||||
|
||||
# ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient)
|
||||
|
||||
def put_file(self, key: str, path: str, content_type: str) -> None:
|
||||
with open(path, "rb") as handle:
|
||||
self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type)
|
||||
|
||||
def download(self, key: str, path: str) -> bool:
|
||||
"""False when the object does not exist."""
|
||||
try:
|
||||
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return False
|
||||
raise
|
||||
with open(path, "wb") as handle:
|
||||
for chunk in response["Body"].iter_chunks(1 << 20):
|
||||
handle.write(chunk)
|
||||
return True
|
||||
|
||||
def copy(self, source_key: str, target_key: str) -> None:
|
||||
"""Server-side copy: the original bytes never pass through the worker again."""
|
||||
self.client.copy_object(
|
||||
Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key},
|
||||
)
|
||||
|
||||
def ready(self) -> bool:
|
||||
try:
|
||||
self.client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,95 @@
|
||||
"""API process entrypoint (lab7): uvicorn behind MANDATORY mutual TLS with an identity allowlist.
|
||||
|
||||
TLS layer (ssl.SSLContext, CERT_REQUIRED): the client must present a certificate chained to the lab root CA.
|
||||
Identity layer (this module, right after the handshake, for every connection including resumed TLS sessions):
|
||||
* DNS SAN must be in ALLOWED_CLIENT_IDENTITIES (only the LB: lb.client.messenger);
|
||||
* the certificate serial must not be in REVOKED_SERIALS_FILE (re-read when it changes, no restart needed).
|
||||
A certificate from the same CA with another identity (worker, api, intruder) is closed before any HTTP byte is read.
|
||||
There is no plaintext listener on the network; the Docker healthcheck uses 127.0.0.1:8081 (app.main).
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
|
||||
import uvicorn
|
||||
from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol
|
||||
|
||||
from app import config, telemetry
|
||||
|
||||
log = logging.getLogger("messenger.mtls")
|
||||
|
||||
|
||||
class Revocations:
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path, self.mtime, self.checked, self.serials = path, None, 0.0, frozenset()
|
||||
|
||||
def current(self) -> frozenset:
|
||||
now = time.monotonic()
|
||||
if self.path and now - self.checked > 1.0:
|
||||
self.checked = now
|
||||
try:
|
||||
mtime = os.stat(self.path).st_mtime
|
||||
if mtime != self.mtime:
|
||||
with open(self.path, encoding="utf-8") as handle:
|
||||
self.serials = frozenset(line.strip().upper() for line in handle if line.strip())
|
||||
self.mtime = mtime
|
||||
log.info("mtls_revocations_loaded count=%d", len(self.serials))
|
||||
except OSError:
|
||||
pass # keep the last known list
|
||||
return self.serials
|
||||
|
||||
|
||||
REVOKED = Revocations(config.REVOKED_SERIALS_FILE)
|
||||
rejected = telemetry.meter.create_counter("messenger.mtls.rejected", unit="{connection}",
|
||||
description="API mTLS connections closed by the identity check, by reason")
|
||||
|
||||
|
||||
def identity_check(cert: dict) -> tuple:
|
||||
"""(allowed, reason, identity, serial) for a validated peer certificate dict (ssl.getpeercert())."""
|
||||
sans = [value for kind, value in cert.get("subjectAltName", ()) if kind == "DNS"]
|
||||
serial = str(cert.get("serialNumber", "")).upper()
|
||||
identity = sans[0] if sans else "-"
|
||||
if serial in REVOKED.current():
|
||||
return False, "revoked", identity, serial
|
||||
if not config.ALLOWED_CLIENT_IDENTITIES.intersection(sans):
|
||||
return False, "identity_not_allowed", identity, serial
|
||||
return True, "ok", identity, serial
|
||||
|
||||
|
||||
class MutualTLSProtocol(HttpToolsProtocol):
|
||||
def connection_made(self, transport) -> None:
|
||||
super().connection_made(transport)
|
||||
ssl_object = transport.get_extra_info("ssl_object")
|
||||
cert = ssl_object.getpeercert() if ssl_object is not None else None
|
||||
allowed, reason, identity, serial = identity_check(cert or {}) if cert else (False, "no_certificate", "-", "-")
|
||||
if not allowed:
|
||||
rejected.add(1, {"reason": reason})
|
||||
log.warning("mtls_client_rejected reason=%s identity=%s serial=%s", reason, identity, serial)
|
||||
transport.abort()
|
||||
|
||||
|
||||
def ssl_context() -> ssl.SSLContext:
|
||||
ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=config.TLS_CA_FILE)
|
||||
ctx.load_cert_chain(config.TLS_SERVER_CERT_FILE, config.TLS_SERVER_KEY_FILE)
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
return ctx
|
||||
|
||||
|
||||
class Config(uvicorn.Config):
|
||||
def load(self) -> None:
|
||||
super().load()
|
||||
self.ssl = ssl_context() # replace uvicorn's context: CA + CERT_REQUIRED + TLS >= 1.2
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg = Config("app.main:app", host=os.environ.get("APP_HOST", "0.0.0.0"), port=int(os.environ.get("APP_PORT", "8080")),
|
||||
http=MutualTLSProtocol, access_log=False, timeout_graceful_shutdown=int(config.GRACEFUL_TIMEOUT),
|
||||
ssl_certfile=config.TLS_SERVER_CERT_FILE, ssl_keyfile=config.TLS_SERVER_KEY_FILE,
|
||||
ssl_ca_certs=config.TLS_CA_FILE, ssl_cert_reqs=ssl.CERT_REQUIRED)
|
||||
uvicorn.Server(cfg).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import valkey.asyncio as valkey
|
||||
|
||||
from app import config, repo
|
||||
from app.cursor import fmt
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, db) -> None:
|
||||
self.db = db
|
||||
self.client = None
|
||||
self.stats = {"hits": 0, "misses": 0, "errors": 0}
|
||||
self._local = {}
|
||||
|
||||
async def open(self) -> None:
|
||||
if not config.CACHE_URL:
|
||||
log.info("session cache disabled: CACHE_URL is empty")
|
||||
return
|
||||
# lab7: rediss:// = TLS to Valkey, server verified against the lab CA (hostname "valkey"); ACL user `api`.
|
||||
self.client = valkey.Valkey.from_url(
|
||||
config.CACHE_URL,
|
||||
socket_timeout=config.CACHE_TIMEOUT,
|
||||
socket_connect_timeout=config.CACHE_TIMEOUT,
|
||||
decode_responses=True,
|
||||
ssl_ca_certs=config.TLS_CA_FILE,
|
||||
ssl_cert_reqs="required",
|
||||
ssl_check_hostname=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
self.client = None
|
||||
|
||||
@staticmethod
|
||||
def digest(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session:" + digest.hex()
|
||||
|
||||
async def create(self, user: dict) -> tuple:
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = self.digest(token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
|
||||
" VALUES (%s, %s, clock_timestamp(), %s)",
|
||||
(digest, user["id"], expires_at),
|
||||
)
|
||||
await self._cache_put(digest, user, expires_at)
|
||||
return token, fmt(expires_at)
|
||||
|
||||
async def resolve(self, token: str) -> Optional[dict]:
|
||||
digest = self.digest(token)
|
||||
cached = await self._cache_get(digest)
|
||||
if cached == REVOKED:
|
||||
self.stats["hits"] += 1
|
||||
return None
|
||||
if cached is not None:
|
||||
self.stats["hits"] += 1
|
||||
return cached
|
||||
self.stats["misses"] += 1
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(digest,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = repo.public_user(row)
|
||||
await self._cache_put(digest, user, row["expires_at"])
|
||||
return user
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
digest = self.digest(token)
|
||||
await self._cache_revoke(digest)
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
|
||||
except Exception:
|
||||
await self._cache_release(digest)
|
||||
raise
|
||||
|
||||
async def sweep(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
return cur.rowcount
|
||||
|
||||
def _cache_ttl(self, expires_at: datetime) -> int:
|
||||
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
|
||||
|
||||
async def _cache_get(self, digest: bytes) -> Optional[dict]:
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("get")
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
if raw == REVOKED:
|
||||
return REVOKED
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
ttl = self._cache_ttl(expires_at)
|
||||
if ttl <= 0:
|
||||
return
|
||||
payload = json.dumps(user)
|
||||
try:
|
||||
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
|
||||
except Exception:
|
||||
self._cache_failed("set")
|
||||
|
||||
async def _cache_release(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.delete(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("release")
|
||||
|
||||
async def _cache_revoke(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
|
||||
except Exception:
|
||||
self._cache_failed("revoke")
|
||||
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
|
||||
|
||||
def _cache_failed(self, operation: str) -> None:
|
||||
self.stats["errors"] += 1
|
||||
if self.stats["errors"] % 50 == 1:
|
||||
log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"])
|
||||
|
||||
# ------------------------------------------------------------------ lab7: shared login rate limit
|
||||
async def hit(self, key: str, window: int) -> tuple:
|
||||
"""INCR a fixed-window counter shared by all replicas. Returns (count, retry_after_seconds).
|
||||
Valkey unavailable -> per-process fallback window (limits stay on, but are not shared)."""
|
||||
if self.client is not None:
|
||||
try:
|
||||
count = await self.client.incr(key)
|
||||
if count == 1:
|
||||
await self.client.expire(key, window)
|
||||
ttl = window
|
||||
else:
|
||||
ttl = await self.client.ttl(key)
|
||||
if ttl < 0: # lost EXPIRE (crash between INCR and EXPIRE): repair
|
||||
await self.client.expire(key, window)
|
||||
ttl = window
|
||||
return count, max(1, ttl)
|
||||
except Exception:
|
||||
self._cache_failed("ratelimit")
|
||||
return self._local_hit(key, window)
|
||||
|
||||
async def peek(self, key: str) -> tuple:
|
||||
if self.client is not None:
|
||||
try:
|
||||
raw = await self.client.get(key)
|
||||
ttl = await self.client.ttl(key) if raw else 0
|
||||
return int(raw or 0), max(1, ttl)
|
||||
except Exception:
|
||||
self._cache_failed("ratelimit")
|
||||
entry = self._local.get(key)
|
||||
if entry and entry[1] > time.monotonic():
|
||||
return entry[0], max(1, int(entry[1] - time.monotonic()))
|
||||
return 0, 1
|
||||
|
||||
async def reset(self, key: str) -> None:
|
||||
self._local.pop(key, None)
|
||||
if self.client is not None:
|
||||
try:
|
||||
await self.client.delete(key)
|
||||
except Exception:
|
||||
self._cache_failed("ratelimit")
|
||||
|
||||
def _local_hit(self, key: str, window: int) -> tuple:
|
||||
now = time.monotonic()
|
||||
count, until = self._local.get(key, (0, now + window))
|
||||
if until <= now:
|
||||
count, until = 0, now + window
|
||||
self._local[key] = (count + 1, until)
|
||||
if len(self._local) > 10000: # bounded memory under a username-spraying attack
|
||||
self._local = {k: v for k, v in self._local.items() if v[1] > now}
|
||||
return count + 1, max(1, int(until - now))
|
||||
@@ -0,0 +1,393 @@
|
||||
"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
|
||||
|
||||
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
|
||||
|
||||
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
|
||||
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
|
||||
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
|
||||
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
|
||||
|
||||
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
|
||||
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
|
||||
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
|
||||
from opentelemetry import context as otel_context
|
||||
from opentelemetry import metrics, propagate, trace
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from app import config
|
||||
|
||||
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
|
||||
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
|
||||
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
QUEUE = config.BROKER_QUEUE
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
|
||||
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
|
||||
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
|
||||
|
||||
tracer = trace.get_tracer("messenger")
|
||||
meter = metrics.get_meter("messenger")
|
||||
_state = {"service": "messenger", "providers": []}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- logging
|
||||
|
||||
class EventFilter(logging.Filter):
|
||||
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
|
||||
|
||||
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not getattr(record, "event", None):
|
||||
match = self.EVENT_RE.match(str(record.msg))
|
||||
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
|
||||
return True
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
out = {
|
||||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
|
||||
"level": record.levelname,
|
||||
"service": _state["service"],
|
||||
"event": getattr(record, "event", ""),
|
||||
"logger": record.name,
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
if ctx.is_valid:
|
||||
out["trace_id"] = format(ctx.trace_id, "032x")
|
||||
out["span_id"] = format(ctx.span_id, "016x")
|
||||
if record.exc_info:
|
||||
out["exc"] = self.formatException(record.exc_info)
|
||||
return json.dumps(out, ensure_ascii=False)
|
||||
|
||||
|
||||
def _setup_logging(logger_provider=None) -> None:
|
||||
root = logging.getLogger()
|
||||
for handler in list(root.handlers):
|
||||
root.removeHandler(handler)
|
||||
root.setLevel(logging.INFO)
|
||||
stdout = logging.StreamHandler(sys.stdout)
|
||||
stdout.setFormatter(JsonFormatter())
|
||||
stdout.addFilter(EventFilter())
|
||||
root.addHandler(stdout)
|
||||
# The exporter's own warnings (backend down) must not be fed back into the exporter.
|
||||
sdk_log = logging.getLogger("opentelemetry")
|
||||
sdk_log.propagate = False
|
||||
sdk_log.addHandler(stdout)
|
||||
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
|
||||
logging.getLogger(noisy).setLevel(logging.WARNING)
|
||||
if logger_provider is not None:
|
||||
from opentelemetry.sdk._logs import LoggingHandler
|
||||
|
||||
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otlp.addFilter(EventFilter())
|
||||
root.addHandler(otlp)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- sampling
|
||||
|
||||
def _build_sampler():
|
||||
"""ParentBased(root=KindFilter(TraceIdRatio)).
|
||||
|
||||
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
|
||||
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
|
||||
Children always follow the parent's decision, so an upload and its worker job are sampled together.
|
||||
"""
|
||||
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
|
||||
TraceIdRatioBased)
|
||||
|
||||
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
||||
decisions = meter.create_counter(
|
||||
"messenger.telemetry.sampler.decisions", unit="{span}",
|
||||
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
|
||||
|
||||
class KindFilter(Sampler):
|
||||
def __init__(self, delegate):
|
||||
self.delegate = delegate
|
||||
|
||||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||||
trace_state=None):
|
||||
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
|
||||
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
|
||||
return SamplingResult(Decision.DROP)
|
||||
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
|
||||
trace_state)
|
||||
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
|
||||
return result
|
||||
|
||||
def get_description(self):
|
||||
return "KindFilter{%s}" % self.delegate.get_description()
|
||||
|
||||
class CountingParentBased(ParentBased):
|
||||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||||
trace_state=None):
|
||||
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
||||
parent = trace.get_current_span(parent_context).get_span_context()
|
||||
if parent.is_valid:
|
||||
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
|
||||
"rule": "parent"})
|
||||
return result
|
||||
|
||||
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
|
||||
|
||||
|
||||
class _CountingExporter:
|
||||
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
|
||||
|
||||
def __init__(self, delegate):
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
|
||||
self.delegate = delegate
|
||||
self.ok = SpanExportResult.SUCCESS
|
||||
self.counter = meter.create_counter(
|
||||
"messenger.telemetry.spans.exported", unit="{span}",
|
||||
description="Spans handed to the OTLP exporter by outcome (success/failure)")
|
||||
|
||||
def export(self, spans):
|
||||
result = self.delegate.export(spans)
|
||||
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
|
||||
return result
|
||||
|
||||
def shutdown(self):
|
||||
return self.delegate.shutdown()
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30000):
|
||||
return self.delegate.force_flush(timeout_millis)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- setup
|
||||
|
||||
def setup(service_name: str, fastapi_app=None) -> None:
|
||||
_state["service"] = service_name
|
||||
if not ENABLED:
|
||||
_setup_logging(None)
|
||||
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
|
||||
return
|
||||
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
|
||||
res = Resource.create({
|
||||
"service.name": service_name,
|
||||
"service.namespace": "messenger",
|
||||
"service.version": SERVICE_VERSION,
|
||||
"service.instance.id": config.INSTANCE_ID,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
})
|
||||
|
||||
views = [
|
||||
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
|
||||
View(instrument_name="http.server.request.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
||||
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
|
||||
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.attachment.time_to_ready",
|
||||
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
||||
# No catch-all View("*"): an instrument matched by two views would be exported twice.
|
||||
# Instruments without a matching view get the SDK default aggregation.
|
||||
]
|
||||
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
||||
views=views)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=res)
|
||||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
_setup_logging(logger_provider)
|
||||
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
|
||||
|
||||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||||
|
||||
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
|
||||
# placeholders), so values, message bodies and hashes never reach db.query.text.
|
||||
BotocoreInstrumentor().instrument()
|
||||
if fastapi_app is not None:
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
|
||||
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
|
||||
exclude_spans=["receive", "send"])
|
||||
_process_metrics()
|
||||
logging.getLogger("messenger.telemetry").info(
|
||||
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
|
||||
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
for provider in _state["providers"]:
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- context carrier
|
||||
|
||||
def inject() -> dict:
|
||||
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
|
||||
carrier: dict = {}
|
||||
if trace.get_current_span().get_span_context().is_valid:
|
||||
propagate.inject(carrier)
|
||||
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
|
||||
|
||||
|
||||
def extract(carrier: Optional[dict]):
|
||||
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
|
||||
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
|
||||
return otel_context.Context()
|
||||
clean = {"traceparent": str(carrier["traceparent"])}
|
||||
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
|
||||
clean["tracestate"] = carrier["tracestate"]
|
||||
return propagate.extract(clean, context=otel_context.Context())
|
||||
|
||||
|
||||
def set_attributes(**attrs) -> None:
|
||||
span = trace.get_current_span()
|
||||
for key, value in attrs.items():
|
||||
if value is not None:
|
||||
span.set_attribute(key.replace("__", "."), value)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
|
||||
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
|
||||
record_exception=True, set_status_on_exception=True) as current:
|
||||
yield current
|
||||
|
||||
|
||||
def record_exception(exc: BaseException) -> None:
|
||||
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
|
||||
current = trace.get_current_span()
|
||||
current.record_exception(exc)
|
||||
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
||||
|
||||
|
||||
def mark_error(current, description: str) -> None:
|
||||
current.set_status(Status(StatusCode.ERROR, description))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------- instruments
|
||||
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
|
||||
|
||||
def _rss_bytes() -> int:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _process_metrics() -> None:
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def cpu(_options):
|
||||
own = resource.getrusage(resource.RUSAGE_SELF)
|
||||
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
|
||||
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
|
||||
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
|
||||
|
||||
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
|
||||
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
|
||||
description="Resident set size of the process")
|
||||
|
||||
|
||||
def register_pool(db) -> None:
|
||||
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def count(_options):
|
||||
stats = db.pool.get_stats()
|
||||
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
|
||||
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
|
||||
yield Observation(idle, {"db.client.connection.state": "idle"})
|
||||
|
||||
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
|
||||
meter.create_observable_up_down_counter(
|
||||
"db.client.connection.pending_requests",
|
||||
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
|
||||
meter.create_observable_up_down_counter(
|
||||
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
|
||||
|
||||
|
||||
def register_queue(state: dict) -> None:
|
||||
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
|
||||
|
||||
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
|
||||
"""
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def depth(_options):
|
||||
queue = state.get("queue")
|
||||
if queue is None:
|
||||
return []
|
||||
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
|
||||
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
|
||||
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
|
||||
|
||||
def oldest(_options):
|
||||
queue = state.get("queue")
|
||||
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
|
||||
|
||||
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
|
||||
description="Jobs accepted but not finished, by state")
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
|
||||
description="Age of the oldest queued/processing job since durable acceptance")
|
||||
|
||||
|
||||
def register_worker(worker) -> None:
|
||||
if not ENABLED:
|
||||
return
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
|
||||
unit="{job}", description="Jobs being processed right now")
|
||||
meter.create_observable_gauge("messenger.worker.capacity",
|
||||
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
|
||||
description="Concurrent job slots of this worker")
|
||||
|
||||
|
||||
jobs_processed = meter.create_counter(
|
||||
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
|
||||
job_duration = meter.create_histogram(
|
||||
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
|
||||
time_to_ready = meter.create_histogram(
|
||||
"messenger.attachment.time_to_ready", unit="s",
|
||||
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")
|
||||
+373
@@ -0,0 +1,373 @@
|
||||
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
|
||||
|
||||
Per delivery:
|
||||
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
|
||||
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
|
||||
3. download staging object, verify size+sha256 against DB metadata
|
||||
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
|
||||
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
|
||||
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
|
||||
7. delete staging (best effort, reconciler finishes it), then ack
|
||||
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
|
||||
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import config, faults, jobs, openbao, telemetry
|
||||
from app.broker import connect, get_queue
|
||||
from app.db import Database
|
||||
from app.s3 import S3Storage
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail or code
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
def __init__(self, code: str, detail: str = ""):
|
||||
super().__init__(code)
|
||||
self.code, self.detail = code, detail or code
|
||||
|
||||
|
||||
class LeaseLost(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def child_limits() -> None:
|
||||
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
cpu = config.JOB_TIMEOUT_SECONDS
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
||||
os.nice(5)
|
||||
|
||||
|
||||
def sha256_file(path: str):
|
||||
digest, size = hashlib.sha256(), 0
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
return digest.hexdigest(), size
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self) -> None:
|
||||
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
|
||||
self.db = Database()
|
||||
self.s3 = S3Storage()
|
||||
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
|
||||
self.stopping = asyncio.Event()
|
||||
self.inflight = 0
|
||||
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
|
||||
|
||||
# ------------------------------------------------------------------ lifecycle
|
||||
|
||||
async def run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, self.stopping.set)
|
||||
await self.db.open()
|
||||
telemetry.register_pool(self.db)
|
||||
telemetry.register_worker(self)
|
||||
await asyncio.to_thread(self.s3.open)
|
||||
bao = openbao.client()
|
||||
if bao is not None:
|
||||
# Rotation of the worker's DB password / S3 key in OpenBao is picked up without a restart.
|
||||
bao.on_change(lambda data: self.db.rotate(data.get("database_url", "")))
|
||||
bao.on_change(self.s3.rotate)
|
||||
bao.start_refresher(config.BAO_REFRESH_SECONDS)
|
||||
connection = await connect()
|
||||
channel = await connection.channel()
|
||||
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
|
||||
queue = await get_queue(channel)
|
||||
tag = await queue.consume(self.on_message, no_ack=False)
|
||||
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
|
||||
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
|
||||
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
|
||||
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
|
||||
await self.stopping.wait()
|
||||
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
|
||||
try:
|
||||
await queue.cancel(tag)
|
||||
except Exception:
|
||||
pass
|
||||
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
|
||||
while self.inflight and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.2)
|
||||
for task in background:
|
||||
task.cancel()
|
||||
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
|
||||
await connection.close()
|
||||
await self.db.close()
|
||||
log.info("worker_stopped stats=%s", self.stats)
|
||||
|
||||
async def alive(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
|
||||
handle.write(str(time.time()))
|
||||
except OSError:
|
||||
pass
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def sweeper(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
|
||||
try:
|
||||
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
|
||||
if swept:
|
||||
log.warning("lease_expired recovered=%s", swept)
|
||||
except Exception:
|
||||
log.warning("lease_sweep_skipped: database unavailable")
|
||||
|
||||
# ------------------------------------------------------------------ delivery
|
||||
|
||||
async def on_message(self, message) -> None:
|
||||
async with self.slots:
|
||||
self.inflight += 1
|
||||
started = time.monotonic()
|
||||
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
|
||||
try:
|
||||
carrier = json.loads(message.body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
carrier = None
|
||||
if not isinstance(carrier, dict) or "traceparent" not in carrier:
|
||||
carrier = dict(message.headers or {})
|
||||
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
|
||||
"messaging.destination.name": config.BROKER_QUEUE,
|
||||
"messaging.message.id": str(message.message_id or ""),
|
||||
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
|
||||
outcome = "requeued"
|
||||
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
|
||||
context=telemetry.extract(carrier), attributes=attrs) as current:
|
||||
try:
|
||||
outcome = await self.handle(message)
|
||||
except Exception:
|
||||
log.exception("unexpected_error, message returned to queue")
|
||||
await asyncio.sleep(1)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
finally:
|
||||
self.inflight -= 1
|
||||
current.set_attribute("messenger.job.outcome", outcome)
|
||||
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
|
||||
telemetry.mark_error(current, outcome)
|
||||
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
|
||||
telemetry.jobs_processed.add(1, labels)
|
||||
telemetry.job_duration.record(time.monotonic() - started, labels)
|
||||
|
||||
@staticmethod
|
||||
async def settle(awaitable) -> None:
|
||||
try:
|
||||
await awaitable
|
||||
except Exception:
|
||||
log.warning("could not settle delivery (connection lost); broker will redeliver")
|
||||
|
||||
async def handle(self, message) -> str:
|
||||
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
|
||||
msg = jobs.parse_message(message.body)
|
||||
if msg is None:
|
||||
self.stats["dlq"] += 1
|
||||
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
|
||||
await self.settle(message.reject(requeue=False))
|
||||
return "dlq"
|
||||
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
|
||||
try:
|
||||
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
|
||||
except Exception:
|
||||
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return "requeued"
|
||||
if decision == "done" and job["state"] == "succeeded":
|
||||
await self.cleanup_staging(job["attachment_id"])
|
||||
if decision != "claimed":
|
||||
self.stats["noop"] += 1
|
||||
telemetry.set_attributes(messenger__job__decision=decision)
|
||||
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
|
||||
message.redelivered)
|
||||
await self.settle(message.ack())
|
||||
return "noop"
|
||||
started = time.monotonic()
|
||||
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
|
||||
job["max_attempts"])
|
||||
lease_lost = asyncio.Event()
|
||||
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
|
||||
outcome = None
|
||||
work = asyncio.create_task(self.process(job, message))
|
||||
lost = asyncio.create_task(lease_lost.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
|
||||
return_when=asyncio.FIRST_COMPLETED)
|
||||
if work not in done:
|
||||
work.cancel()
|
||||
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
|
||||
outcome = "succeeded" if work.result() else "lost"
|
||||
except LeaseLost:
|
||||
outcome = "lost"
|
||||
except Permanent as exc:
|
||||
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
|
||||
except Transient as exc:
|
||||
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
|
||||
except Exception as exc: # S3/DB/network errors
|
||||
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
|
||||
finally:
|
||||
keeper.cancel()
|
||||
lost.cancel()
|
||||
if outcome is None:
|
||||
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return "requeued"
|
||||
self.stats[outcome] = self.stats.get(outcome, 0) + 1
|
||||
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
|
||||
time.monotonic() - started)
|
||||
await self.settle(message.ack())
|
||||
return outcome
|
||||
|
||||
async def cleanup_staging(self, attachment_id) -> None:
|
||||
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
src = await jobs.attachment_source(conn, attachment_id)
|
||||
if src and src["status"] == "ready" and src["staging_key"]:
|
||||
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
||||
except Exception:
|
||||
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
|
||||
|
||||
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
|
||||
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
|
||||
except Exception:
|
||||
continue # DB blip: keep trying until the lease really expires
|
||||
if not alive:
|
||||
log.warning("lease_lost job=%s: another worker owns it now", job_id)
|
||||
lease_lost.set()
|
||||
return
|
||||
|
||||
async def record_failure(self, job, code: str, detail: str, transient: bool):
|
||||
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
|
||||
transient)
|
||||
try:
|
||||
return await self.db.in_transaction(
|
||||
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
|
||||
except Exception:
|
||||
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------ the actual work
|
||||
|
||||
async def process(self, job, message) -> bool:
|
||||
async with self.db.connection() as conn:
|
||||
src = await jobs.attachment_source(conn, job["attachment_id"])
|
||||
if src is None or not src["staging_key"]:
|
||||
raise Permanent("staging_missing", "attachment has no staging object")
|
||||
filename = src["filename"]
|
||||
first_delivery = job["attempt"] == 1 and not message.redelivered
|
||||
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
|
||||
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
|
||||
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
|
||||
if faults.active(filename, "permanent"):
|
||||
raise Permanent("invalid_image", "injected permanent failure")
|
||||
if job["attempt"] <= faults.transient_until(filename):
|
||||
raise Transient("dependency_unavailable", "injected transient failure")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
||||
source = os.path.join(tmp, "source")
|
||||
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
|
||||
raise Permanent("staging_missing", "staging object not found")
|
||||
digest, size = await asyncio.to_thread(sha256_file, source)
|
||||
if digest != src["sha256"] or size != src["size_bytes"]:
|
||||
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
|
||||
out_dir = os.path.join(tmp, "out")
|
||||
os.mkdir(out_dir)
|
||||
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
|
||||
produced = await self.run_child(source, out_dir)
|
||||
|
||||
variants = []
|
||||
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
|
||||
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
|
||||
for item in produced:
|
||||
digest, size = await asyncio.to_thread(sha256_file, item["file"])
|
||||
key = prefix + os.path.basename(item["file"])
|
||||
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
|
||||
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
|
||||
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
|
||||
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
|
||||
raise Permanent("processing_failed", "not all variants were produced")
|
||||
|
||||
if first_delivery and faults.active(filename, "worker_crash_after_put"):
|
||||
faults.crash("after S3 put, before result COMMIT")
|
||||
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
|
||||
if not committed:
|
||||
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
|
||||
return False
|
||||
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
|
||||
{"job.type": telemetry.JOB_TYPE})
|
||||
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
|
||||
faults.crash("after result COMMIT, before ack")
|
||||
try:
|
||||
await asyncio.to_thread(self.s3.delete, src["staging_key"])
|
||||
except Exception:
|
||||
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
|
||||
return True
|
||||
|
||||
async def run_child(self, source: str, out_dir: str) -> list:
|
||||
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
|
||||
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
|
||||
)
|
||||
try:
|
||||
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
|
||||
except asyncio.CancelledError:
|
||||
proc.kill()
|
||||
raise
|
||||
code = proc.returncode
|
||||
if code == 0:
|
||||
return json.loads(stdout)
|
||||
if code in PERMANENT_EXIT:
|
||||
raise Permanent(PERMANENT_EXIT[code])
|
||||
if code == -signal.SIGXCPU:
|
||||
raise Permanent("image_too_complex", "CPU limit exceeded")
|
||||
if code == -signal.SIGKILL:
|
||||
raise Transient("worker_resource_exhausted", "child killed (OOM)")
|
||||
raise Permanent("processing_failed", "child exit %s" % code)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
telemetry.setup("messenger-worker")
|
||||
try:
|
||||
asyncio.run(Worker().run())
|
||||
finally:
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+604
@@ -0,0 +1,604 @@
|
||||
name: mtusi-lab7
|
||||
# Trust layout: contracts/trust-matrix.md. Every TLS key reaches only its own container (Docker secrets from
|
||||
# secrets/pki, created by scripts/pki.sh). API and worker receive NO passwords: they authenticate to OpenBao with
|
||||
# their client certificate and read their scoped KV secret (scripts/bao.sh bootstrap).
|
||||
|
||||
x-hardened: &hardened
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
restart: unless-stopped
|
||||
|
||||
x-otel-env: &otel-env
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/otel.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/otel.key
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||
OTEL_SEMCONV_STABILITY_OPT_IN: http
|
||||
OTEL_METRIC_EXPORT_INTERVAL: "10000"
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
|
||||
OTEL_BSP_SCHEDULE_DELAY: "2000"
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
|
||||
SERVICE_VERSION: "1.7.0"
|
||||
|
||||
x-app-env: &app-env
|
||||
TLS_CA_FILE: /run/secrets/ca.crt
|
||||
TLS_CLIENT_CERT_FILE: /run/secrets/client.crt
|
||||
TLS_CLIENT_KEY_FILE: /run/secrets/client.key
|
||||
BAO_ADDR: https://openbao:8200
|
||||
S3_ENDPOINT: https://minio:9000
|
||||
BROKER_URL: amqps://rabbitmq:5671/messenger?auth=external&heartbeat=30
|
||||
DATABASE_SSLMODE: verify-full
|
||||
|
||||
x-api: &api
|
||||
<<: *hardened
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab7
|
||||
env_file: [.env]
|
||||
tmpfs: ["/tmp:size=128m,mode=1777"]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
valkey: {condition: service_healthy}
|
||||
openbao: {condition: service_healthy}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 512M, pids: 256}
|
||||
# Business port 8080 is mTLS-only; the healthcheck uses the loopback-only health listener 127.0.0.1:8081.
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).status == 200 else 1)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
services:
|
||||
nginx:
|
||||
<<: *hardened
|
||||
image: nginx:1.30.6-alpine@sha256:583b7f2fce1cdb8da5ad5c39ebd3c33b98b87d493190e548bdee1fa8c619c311
|
||||
user: "101:101"
|
||||
command: ["/bin/sh", "/opt/nginx-health.sh"]
|
||||
environment:
|
||||
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
|
||||
HEALTH_INTERVAL: "1"
|
||||
HEALTH_FALL: "2"
|
||||
HEALTH_RISE: "2"
|
||||
ports:
|
||||
- "127.0.0.1:8443:8443" # the only business ingress: TLS
|
||||
tmpfs: ["/tmp:uid=101,gid=101,mode=0700", "/var/cache/nginx:uid=101,gid=101,mode=0700",
|
||||
"/var/run:uid=101,gid=101,mode=0755", "/etc/nginx/upstream:uid=101,gid=101,mode=0755"]
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: lb_server_crt, target: server.crt}
|
||||
- {source: lb_server_key, target: server.key}
|
||||
- {source: lb_client_crt, target: client.crt}
|
||||
- {source: lb_client_key, target: client.key}
|
||||
depends_on:
|
||||
api-1: {condition: service_healthy}
|
||||
api-2: {condition: service_healthy}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 64M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://localhost:8443/health/live >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
|
||||
api-1:
|
||||
<<: *api
|
||||
environment:
|
||||
<<: [*otel-env, *app-env]
|
||||
INSTANCE_ID: api-1
|
||||
LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
|
||||
BAO_AUTH_ROLE: api
|
||||
BAO_KV_PATH: api
|
||||
TLS_SERVER_CERT_FILE: /run/secrets/server.crt
|
||||
TLS_SERVER_KEY_FILE: /run/secrets/server.key
|
||||
ALLOWED_CLIENT_IDENTITIES: lb.client.messenger
|
||||
REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||||
secrets: &api-secrets
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: api_1_crt, target: server.crt}
|
||||
- {source: api_1_key, target: server.key}
|
||||
- {source: api_client_crt, target: client.crt}
|
||||
- {source: api_client_key, target: client.key}
|
||||
- {source: api_otel_crt, target: otel.crt}
|
||||
- {source: api_otel_key, target: otel.key}
|
||||
- {source: revoked_serials, target: revoked-serials.txt}
|
||||
|
||||
api-2:
|
||||
<<: *api
|
||||
environment:
|
||||
<<: [*otel-env, *app-env]
|
||||
INSTANCE_ID: api-2
|
||||
LOGIN_RATE_LIMIT_PER_IP: ${LOGIN_RATE_LIMIT_PER_IP:-120}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4}
|
||||
BAO_AUTH_ROLE: api
|
||||
BAO_KV_PATH: api
|
||||
TLS_SERVER_CERT_FILE: /run/secrets/server.crt
|
||||
TLS_SERVER_KEY_FILE: /run/secrets/server.key
|
||||
ALLOWED_CLIENT_IDENTITIES: lb.client.messenger
|
||||
REVOKED_SERIALS_FILE: /run/secrets/revoked-serials.txt
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: api_2_crt, target: server.crt}
|
||||
- {source: api_2_key, target: server.key}
|
||||
- {source: api_client_crt, target: client.crt}
|
||||
- {source: api_client_key, target: client.key}
|
||||
- {source: api_otel_crt, target: otel.crt}
|
||||
- {source: api_otel_key, target: otel.key}
|
||||
- {source: revoked_serials, target: revoked-serials.txt}
|
||||
|
||||
worker:
|
||||
<<: *hardened
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab7
|
||||
command: ["python", "-m", "app.worker"]
|
||||
env_file: [.env]
|
||||
environment:
|
||||
<<: [*otel-env, *app-env]
|
||||
INSTANCE_ID: ""
|
||||
BAO_AUTH_ROLE: worker
|
||||
BAO_KV_PATH: worker
|
||||
DB_POOL_MAX: "6"
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
tmpfs: ["/tmp:size=512m,mode=1777"]
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: worker_client_crt, target: client.crt}
|
||||
- {source: worker_client_key, target: client.key}
|
||||
- {source: worker_otel_crt, target: otel.crt}
|
||||
- {source: worker_otel_key, target: otel.key}
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
openbao: {condition: service_healthy}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "${WORKER_CPUS:-2.00}", memory: 1536M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
migrate:
|
||||
<<: *hardened
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab7
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
restart: "no"
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/migrate_database_url
|
||||
DATABASE_SSLMODE: verify-full
|
||||
TLS_CA_FILE: /run/secrets/ca.crt
|
||||
DB_API_PASSWORD_FILE: /run/secrets/db_api_password
|
||||
DB_WORKER_PASSWORD_FILE: /run/secrets/db_worker_password
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- migrate_database_url
|
||||
- db_api_password
|
||||
- db_worker_password
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
|
||||
openbao:
|
||||
<<: *hardened
|
||||
image: openbao/openbao:2.6.5@sha256:09f1eaaee351c329d75a5a89654d6e1ad4792998650c3cbf0c3513a76d9d9b6e
|
||||
user: "100:1000"
|
||||
entrypoint: ["bao", "server", "-config=/openbao/config/config.hcl"]
|
||||
tmpfs: ["/tmp"]
|
||||
volumes:
|
||||
- ./infra/openbao/config.hcl:/openbao/config/config.hcl:ro
|
||||
- baodata:/openbao/file
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: openbao_crt, target: openbao.crt}
|
||||
- {source: openbao_key, target: openbao.key}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
# Healthy = server answers (sealed or not). Services retry their login while it is sealed.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/run/secrets/ca.crt bao status >/dev/null 2>&1; [ $$? -ne 1 ]"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
postgres:
|
||||
<<: *hardened
|
||||
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
|
||||
user: "70:70"
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: --data-checksums --auth-host=scram-sha-256
|
||||
# PostgreSQL refuses a key file readable by others: copy it to the tmpfs with mode 0600 first.
|
||||
entrypoint: ["sh", "-c", "install -m 600 /run/secrets/server.key /tmp/server.key && exec docker-entrypoint.sh \"$$@\"", "--"]
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- ssl=on
|
||||
- -c
|
||||
- ssl_cert_file=/run/secrets/server.crt
|
||||
- -c
|
||||
- ssl_key_file=/tmp/server.key
|
||||
- -c
|
||||
- ssl_min_protocol_version=TLSv1.2
|
||||
- -c
|
||||
- hba_file=/etc/postgresql/pg_hba.conf
|
||||
- -c
|
||||
- password_encryption=scram-sha-256
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
- -c
|
||||
- max_connections=60
|
||||
tmpfs: ["/tmp", "/var/run/postgresql"]
|
||||
secrets:
|
||||
- postgres_password
|
||||
- {source: postgres_crt, target: server.crt}
|
||||
- {source: postgres_key, target: server.key}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
- ./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h /var/run/postgresql"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
|
||||
valkey:
|
||||
<<: *hardened
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
user: "999:1000"
|
||||
entrypoint: ["sh", "/opt/valkey-start.sh"]
|
||||
tmpfs: ["/tmp", "/data"]
|
||||
volumes:
|
||||
- ./infra/valkey/valkey-start.sh:/opt/valkey-start.sh:ro
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: valkey_crt, target: server.crt}
|
||||
- {source: valkey_key, target: server.key}
|
||||
- valkey_api_password
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "--tls", "--cacert", "/run/secrets/ca.crt", "--user", "health", "--pass", "health", "-h", "valkey", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
rabbitmq:
|
||||
<<: *hardened
|
||||
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
|
||||
user: "100:101"
|
||||
hostname: rabbitmq
|
||||
environment:
|
||||
RABBITMQ_ENABLED_PLUGINS_FILE: /etc/rabbitmq/enabled_plugins
|
||||
# Erlang distribution + epmd loopback-only (no cluster; CLI runs inside the container). The node name must
|
||||
# then resolve to loopback, hence rabbit@localhost (its data dir is /var/lib/rabbitmq/mnesia/rabbit@localhost).
|
||||
RABBITMQ_NODENAME: rabbit@localhost
|
||||
ERL_EPMD_ADDRESS: 127.0.0.1
|
||||
tmpfs: ["/tmp", "/etc/rabbitmq/conf.d"]
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
|
||||
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: rabbitmq_crt, target: server.crt}
|
||||
- {source: rabbitmq_key, target: server.key}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
|
||||
minio:
|
||||
<<: *hardened
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
command: ["server", "/data", "--certs-dir", "/tmp/certs"]
|
||||
entrypoint: ["sh", "-c", "mkdir -p /tmp/certs/CAs && cp /run/secrets/server.crt /tmp/certs/public.crt && cp /run/secrets/server.key /tmp/certs/private.key && cp /run/secrets/ca.crt /tmp/certs/CAs/ca.crt && exec minio \"$$@\"", "--"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
tmpfs: ["/tmp"]
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: minio_crt, target: server.crt}
|
||||
- {source: minio_key, target: server.key}
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 512M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "mkdir -p /tmp/hc/certs/CAs && cp /run/secrets/ca.crt /tmp/hc/certs/CAs/ca.crt && MC_HOST_hc=https://minio:9000 mc --config-dir /tmp/hc ready hc >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
|
||||
minio-init:
|
||||
<<: *hardened
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
entrypoint: ["/bin/sh", "/opt/init.sh"]
|
||||
restart: "no"
|
||||
environment:
|
||||
S3_BUCKET: messenger-attachments
|
||||
MC_CONFIG_DIR: /tmp/mc
|
||||
tmpfs: ["/tmp"]
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- s3_api_access_key
|
||||
- s3_api_secret_key
|
||||
- s3_worker_access_key
|
||||
- s3_worker_secret_key
|
||||
volumes:
|
||||
- ./infra/minio/init.sh:/opt/init.sh:ro
|
||||
- ./infra/minio/policy-api.json:/etc/minio-policy/policy-api.json:ro
|
||||
- ./infra/minio/policy-worker.json:/etc/minio-policy/policy-worker.json:ro
|
||||
depends_on:
|
||||
minio: {condition: service_healthy}
|
||||
|
||||
# ------------------------------------------------------------------ observability (lab6) behind mTLS
|
||||
otel-collector:
|
||||
<<: *hardened
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917
|
||||
command:
|
||||
- --config=/etc/otelcol/collector.yaml
|
||||
- --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
- ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro
|
||||
- ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: ingest_ca_crt, target: ingest-ca.crt}
|
||||
- {source: otel_collector_crt, target: server.crt}
|
||||
- {source: otel_collector_key, target: server.key}
|
||||
- {source: collector_out_crt, target: client.crt}
|
||||
- {source: collector_out_key, target: client.key}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 256M, pids: 256}
|
||||
|
||||
prometheus:
|
||||
<<: *hardened
|
||||
image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --web.config.file=/etc/prometheus/web.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --web.enable-otlp-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro
|
||||
- ./infra/prometheus/health-client.yml:/etc/prometheus/health-client.yml:ro
|
||||
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||
- promdata:/prometheus
|
||||
secrets:
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
|
||||
- {source: prometheus_crt, target: server.crt}
|
||||
- {source: prometheus_key, target: server.key}
|
||||
- {source: operator_crt, target: health.crt}
|
||||
- {source: operator_key, target: health.key}
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090" # mTLS: operator certificate only
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD", "promtool", "check", "healthy", "--url=https://prometheus:9090", "--http.config.file=/etc/prometheus/health-client.yml"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
loki:
|
||||
<<: *hardened
|
||||
image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
secrets:
|
||||
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
|
||||
- {source: loki_crt, target: server.crt}
|
||||
- {source: loki_key, target: server.key}
|
||||
ports:
|
||||
- "127.0.0.1:3100:3100" # mTLS: operator certificate only
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M, pids: 256}
|
||||
healthcheck: # TCP only: every HTTP request needs a telemetry client certificate
|
||||
test: ["CMD", "/busybox/nc", "-z", "127.0.0.1", "3100"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
|
||||
tempo:
|
||||
<<: *hardened
|
||||
image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
tmpfs: ["/tmp"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
secrets:
|
||||
- {source: telemetry_ca_crt, target: telemetry-ca.crt}
|
||||
- {source: tempo_crt, target: server.crt}
|
||||
- {source: tempo_key, target: server.key}
|
||||
ports:
|
||||
- "127.0.0.1:3200:3200" # mTLS: operator certificate only
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 1024M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD", "nc", "-z", "127.0.0.1", "3200"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
|
||||
grafana:
|
||||
<<: *hardened
|
||||
image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7
|
||||
environment:
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/secrets/server.crt
|
||||
GF_SERVER_CERT_KEY: /run/secrets/server.key
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_SECURITY_COOKIE_SECURE: "true"
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
|
||||
tmpfs: ["/tmp"]
|
||||
secrets:
|
||||
- grafana_admin_password
|
||||
- {source: ca_crt, target: ca.crt}
|
||||
- {source: grafana_crt, target: server.crt}
|
||||
- {source: grafana_key, target: server.key}
|
||||
- {source: grafana_out_crt, target: client.crt}
|
||||
- {source: grafana_out_key, target: client.key}
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000" # HTTPS + admin login
|
||||
depends_on:
|
||||
prometheus: {condition: service_healthy}
|
||||
loki: {condition: service_started}
|
||||
tempo: {condition: service_started}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 256M, pids: 256}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf --cacert /run/secrets/ca.crt https://grafana:3000/api/health >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
miniodata:
|
||||
rabbitmqdata:
|
||||
baodata:
|
||||
promdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
|
||||
secrets:
|
||||
ca_crt: {file: ./secrets/pki/ca.crt}
|
||||
ingest_ca_crt: {file: ./secrets/pki/ingest.crt}
|
||||
telemetry_ca_crt: {file: ./secrets/pki/telemetry.crt}
|
||||
revoked_serials: {file: ./secrets/pki/revoked-serials.txt}
|
||||
lb_server_crt: {file: ./secrets/pki/lb-server.crt}
|
||||
lb_server_key: {file: ./secrets/pki/lb-server.key}
|
||||
lb_client_crt: {file: ./secrets/pki/lb-client.crt}
|
||||
lb_client_key: {file: ./secrets/pki/lb-client.key}
|
||||
api_1_crt: {file: ./secrets/pki/api-1.crt}
|
||||
api_1_key: {file: ./secrets/pki/api-1.key}
|
||||
api_2_crt: {file: ./secrets/pki/api-2.crt}
|
||||
api_2_key: {file: ./secrets/pki/api-2.key}
|
||||
api_client_crt: {file: ./secrets/pki/api.crt}
|
||||
api_client_key: {file: ./secrets/pki/api.key}
|
||||
worker_client_crt: {file: ./secrets/pki/worker.crt}
|
||||
worker_client_key: {file: ./secrets/pki/worker.key}
|
||||
api_otel_crt: {file: ./secrets/pki/api-otel.crt}
|
||||
api_otel_key: {file: ./secrets/pki/api-otel.key}
|
||||
worker_otel_crt: {file: ./secrets/pki/worker-otel.crt}
|
||||
worker_otel_key: {file: ./secrets/pki/worker-otel.key}
|
||||
postgres_crt: {file: ./secrets/pki/postgres.crt}
|
||||
postgres_key: {file: ./secrets/pki/postgres.key}
|
||||
minio_crt: {file: ./secrets/pki/minio.crt}
|
||||
minio_key: {file: ./secrets/pki/minio.key}
|
||||
rabbitmq_crt: {file: ./secrets/pki/rabbitmq.crt}
|
||||
rabbitmq_key: {file: ./secrets/pki/rabbitmq.key}
|
||||
valkey_crt: {file: ./secrets/pki/valkey.crt}
|
||||
valkey_key: {file: ./secrets/pki/valkey.key}
|
||||
openbao_crt: {file: ./secrets/pki/openbao.crt}
|
||||
openbao_key: {file: ./secrets/pki/openbao.key}
|
||||
otel_collector_crt: {file: ./secrets/pki/otel-collector.crt}
|
||||
otel_collector_key: {file: ./secrets/pki/otel-collector.key}
|
||||
collector_out_crt: {file: ./secrets/pki/collector-out.crt}
|
||||
collector_out_key: {file: ./secrets/pki/collector-out.key}
|
||||
prometheus_crt: {file: ./secrets/pki/prometheus.crt}
|
||||
prometheus_key: {file: ./secrets/pki/prometheus.key}
|
||||
loki_crt: {file: ./secrets/pki/loki.crt}
|
||||
loki_key: {file: ./secrets/pki/loki.key}
|
||||
tempo_crt: {file: ./secrets/pki/tempo.crt}
|
||||
tempo_key: {file: ./secrets/pki/tempo.key}
|
||||
grafana_crt: {file: ./secrets/pki/grafana.crt}
|
||||
grafana_key: {file: ./secrets/pki/grafana.key}
|
||||
grafana_out_crt: {file: ./secrets/pki/grafana-out.crt}
|
||||
grafana_out_key: {file: ./secrets/pki/grafana-out.key}
|
||||
operator_crt: {file: ./secrets/pki/operator.crt}
|
||||
operator_key: {file: ./secrets/pki/operator.key}
|
||||
postgres_password: {file: ./secrets/bootstrap/postgres_password}
|
||||
migrate_database_url: {file: ./secrets/bootstrap/migrate_database_url}
|
||||
db_api_password: {file: ./secrets/bootstrap/db_api_password}
|
||||
db_worker_password: {file: ./secrets/bootstrap/db_worker_password}
|
||||
minio_root_user: {file: ./secrets/bootstrap/minio_root_user}
|
||||
minio_root_password: {file: ./secrets/bootstrap/minio_root_password}
|
||||
s3_api_access_key: {file: ./secrets/bootstrap/s3_api_access_key}
|
||||
s3_api_secret_key: {file: ./secrets/bootstrap/s3_api_secret_key}
|
||||
s3_worker_access_key: {file: ./secrets/bootstrap/s3_worker_access_key}
|
||||
s3_worker_secret_key: {file: ./secrets/bootstrap/s3_worker_secret_key}
|
||||
valkey_api_password: {file: ./secrets/bootstrap/valkey_api_password}
|
||||
grafana_admin_password: {file: ./secrets/bootstrap/grafana_admin_password}
|
||||
+73
-12
@@ -4,18 +4,18 @@
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | `infra/nginx.conf` (listen 8443 ssl), `app/main.py` login; аудит I1–I4, `make test-security`, `test_auth_acl.py` |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | `app/serve.py` (CERT_REQUIRED + SAN allowlist + отозванные serial), `infra/nginx.conf` proxy_ssl_*; аудит A1–A14, `test_cert_rotation.py` |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | `infra/postgres/pg_hba.conf`, `migrations/0007_service_roles.sql`, `app/db.py` (verify-full); аудит B1–B14, `test_secret_rotation.py` |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | `infra/minio/policy-*.json`, `app/s3.py` (verify=CA); аудит C1–C14 |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | `infra/rabbitmq/rabbitmq.conf`, `definitions.json`, `app/broker.py`; аудит D1–D14 |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | `infra/otel/collector.yaml` (client_ca_file = ingest CA); аудит E1–E7 |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | `infra/prometheus/web.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, exporters в `collector.yaml`; аудит F1–F24 |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | `infra/grafana/provisioning/datasources/datasources.yaml` (tlsAuth + CA); аудит F (grafana-out), E (grafana-out отклонён), J1–J2 |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | `infra/openbao/config.hcl`, `policies/*.hcl`, `scripts/bao.sh`, `app/openbao.py`; аудит H1–H20, `test_bao_sealed.py` |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | `scripts/bao.sh`, `compose.yaml` (порты только 127.0.0.1); аудит J, H (admin-пути запрещены приложениям), `check_hardening.py` |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | `infra/valkey/valkey-start.sh` (TLS, ACL), `app/sessions.py`; аудит G1–G11 |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | N/A: репликации и кластеров нет (по одному узлу каждого хранилища). Erlang distribution RabbitMQ, внутренний gRPC Loki/Tempo — только loopback; аудит K1–K13 |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
|
||||
@@ -35,3 +35,64 @@
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
|
||||
|
||||
---
|
||||
|
||||
# Заполнение (решение lab7)
|
||||
|
||||
Все отрицательные и положительные проверки ниже выполняет `student-tests/tls_audit.py` изнутри сети стенда
|
||||
(разделы A–K, **135 проверок**). Сырой вывод: `evidence/tls_audit.txt`, `evidence/tls_audit.json`.
|
||||
Запуск: `docker compose run --rm --no-deps -v ./secrets/pki:/pki:ro -v ./secrets/bootstrap:/bootstrap:ro -v ./student-tests:/audit:ro --entrypoint python worker /audit/tls_audit.py`.
|
||||
|
||||
## PKI и identities
|
||||
|
||||
| CA | Кем подписан | Что подписывает | Кто доверяет |
|
||||
| --- | --- | --- | --- |
|
||||
| Messenger Lab Root CA (`ca.crt`, EC P-256, 825 дн.) | self-signed | все серверные сертификаты; клиентские identities `lb`, `api`, `worker`, `intruder` (тестовый) | все клиенты для проверки сервера; API (LB), RabbitMQ, OpenBao, PostgreSQL |
|
||||
| Telemetry Ingest CA | Root | только `api.otel.messenger`, `worker.otel.messenger` | **только** Collector (client_ca_file) |
|
||||
| Telemetry Clients CA | Root | только `collector.out`, `grafana.out`, `operator` | **только** Prometheus, Loki, Tempo |
|
||||
| Foreign CA (тестовый) | self-signed | `foreign-client`, заявляет имена lb/api/worker | никто |
|
||||
|
||||
Leaf-сертификаты живут 90 дней (`scripts/pki.sh`), ключи EC P-256 лежат только в `secrets/pki/` (0700) и попадают в контейнер как Docker secret именно того сервиса. Отзыв: `revoked-serials.txt`, API перечитывает его на лету.
|
||||
|
||||
## Детальная матрица
|
||||
|
||||
| Связь | Порт / endpoint | Сервер (CA, SAN) | Клиентская identity | Проверка SAN / ACL | Срок жизни credentials | Положительный / отрицательный тест |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Клиент → nginx | `127.0.0.1:8443` HTTPS | lb-server: Root, SAN `localhost`, `nginx`, `127.0.0.1` | пользователь: Bearer (256 бит, в БД SHA-256), вход Basic | сессия проверяется на каждом запросе (PostgreSQL, кэш Valkey ≤ 30 с и не дольше expires_at); logout = отзыв в кэше и БД; лимит входа 5 неудач на username и 120 попыток на IP за 60 с, общий для реплик (Valkey) | сессия 3600 с; сертификат 90 дн. | `make test`, `make test-security`; аудит I: чужой CA и неверный hostname → ошибка проверки сертификата; `test_auth_acl.py`: logout, expiry, 429 + Retry-After |
|
||||
| nginx → API-1/2 | `api-N:8080` HTTPS, mTLS; plaintext-listener нет | api-N: Root, SAN `api-N`, `api.internal` (nginx проверяет `api.internal`) | `lb.client.messenger` (Root, EKU clientAuth) | `app/serve.py`: CERT_REQUIRED (цепочка до Root) **и** DNS SAN ∈ `ALLOWED_CLIENT_IDENTITIES` **и** serial ∉ revoked; проверяется на каждом соединении, включая возобновлённые TLS-сессии | 90 дн.; отзыв ≤ 1 с для новых соединений | аудит A: без сертификата, чужой CA, `intruder`/`worker`/`api` от Root → соединение закрыто; `lb` → 200. `test_cert_rotation.py`: ротация, отзыв старого serial, просроченный |
|
||||
| API/worker → PostgreSQL | `postgres:5432`, `sslmode=verify-full` | postgres: Root, SAN `postgres` | роли `app_api`, `app_worker` (SCRAM-SHA-256) | `pg_hba`: только `hostssl … scram-sha-256`, `host … reject`; права по таблицам: worker без users/messages/sessions и без DELETE, API без DDL и ALTER ROLE | пароль в OpenBao KV; ротация `bao.sh rotate-worker-db` | аудит B: plaintext, чужой CA, неверный hostname, неверный пароль → отказ; 9 проверок allow/deny прав |
|
||||
| API/worker → MinIO | `minio:9000` HTTPS | minio: Root, SAN `minio` | `messenger-api`, `messenger-worker` | API: PUT/GET/DELETE `staging/*`, GET/DELETE `attachments/*`, LIST; worker: GET/DELETE `staging/*`, только PUT `attachments/*` | ключи в OpenBao KV (callback `s3.rotate`) | аудит C: чужой CA, plaintext → отказ; worker: GET/DELETE `attachments`, PUT `staging`, LIST, чужой префикс → AccessDenied; API: PUT `attachments`, создание бакета → AccessDenied |
|
||||
| API → RabbitMQ (producer) | `rabbitmq:5671` AMQPS, mTLS; 5672 закрыт | rabbitmq: Root, SAN `rabbitmq` | `api.client.messenger` (EXTERNAL = CN) | permissions: configure `^$`, write `^attachments$`, read `^$` | 90 дн. | аудит D: без сертификата, чужой CA, `intruder`, `lb` → отказ; API: publish разрешён; consume, declare и default exchange → ACCESS_REFUSED |
|
||||
| Worker → RabbitMQ (consumer) | то же | то же | `worker.client.messenger` | configure `^$`, write `^$`, read `^attachments\.process$` | 90 дн. | worker: consume разрешён; publish и declare → ACCESS_REFUSED |
|
||||
| API/worker → Collector | `otel-collector:4318` OTLP/HTTPS, mTLS; gRPC 4317 выключен | otel-collector: Root | `api.otel.messenger` / `worker.otel.messenger` (Ingest CA) | client_ca_file = только Ingest CA: сертификаты Root-CA и Telemetry Clients CA отклоняются | 90 дн. | аудит E: без сертификата, чужой CA, `lb`/`api` (Root), `grafana-out` → отказ; api-otel и worker-otel → 200 |
|
||||
| Collector → Prometheus / Loki / Tempo | `prometheus:9090` `/api/v1/otlp`, `loki:3100` `/otlp`, `tempo:4317` gRPC — все с TLS | Root, SAN по имени сервиса | `collector.out.messenger` (Telemetry Clients CA) | бэкенды доверяют только Telemetry Clients CA | 90 дн. | аудит F: без сертификата, чужой CA, `api` (Root), `api-otel` (Ingest) → отказ; collector-out, grafana-out, operator → 200 |
|
||||
| Grafana → источники | те же HTTPS-порты | проверка сервера по Root | `grafana.out.messenger` | Telemetry Clients CA; к Collector и API не допускается | 90 дн. | аудит F (accept), E (grafana-out в Collector → отказ); запросы панелей через Grafana: `evidence/dashboard_check.txt` |
|
||||
| API/worker → OpenBao | `openbao:8200` HTTPS | openbao: Root, SAN `openbao` | сертификат `api` / `worker` → cert auth (роль = ровно этот CN) | политика `messenger-<role>`: только `read` своего `messenger/data/<role>` и metadata | токен 15 мин, max 1 ч; renew/re-login каждые 15 с | аудит H: без сертификата, `intruder`, чужая роль, чужой CA → нет токена; чужой путь, запись, изменение политики, admin-пути → 403 |
|
||||
| Оператор → OpenBao | не опубликован; `docker compose exec` + CLI по TLS на 127.0.0.1 | Root | root token и 3 доли unseal в `secrets/operator/` (0600) | у приложений нет admin-прав (аудит H: `sys/auth` → 403) | root token живёт у оператора; доли нужны при каждом рестарте OpenBao | `scripts/bao.sh status|unseal|seal` |
|
||||
| Оператор → Grafana | `127.0.0.1:3000` HTTPS | grafana: Root, SAN `localhost` | admin-пароль (Docker secret) | anonymous выключен, регистрация выключена | — | аудит J: без логина → 401; plaintext → 400 |
|
||||
| Оператор → MinIO admin | не опубликован, консоль выключена | — | root-ключи только у `minio-init` (bootstrap) | приложения используют только scoped-пользователей | — | аудит K (порт 9001 закрыт), C (у приложений нет createBucket) |
|
||||
| Оператор → Prometheus/Loki/Tempo | `127.0.0.1:9090/3100/3200` | Root | `operator.messenger` (Telemetry Clients CA) | mTLS | 90 дн. | аудит F (operator → 200, без сертификата → отказ) |
|
||||
| API → Valkey | `valkey:6379` TLS (plaintext port 0) | valkey: Root, SAN `valkey` | ACL-пользователь `api` (пароль из OpenBao) | ключи только `session:*` и `ratelimit:*`; команды get/set/del/incr/incrby/expire/ttl/ping; `default` выключен | пароль в OpenBao | аудит G: plaintext, чужой CA, без аутентификации, неверный пароль → отказ; другие ключи, FLUSHALL, CONFIG, KEYS → NOPERM |
|
||||
| Replication / кластер | — | — | — | N/A: по одному узлу | — | аудит K: 25672/4369 (RabbitMQ), 8201 (OpenBao) закрыты |
|
||||
|
||||
## Служебные каналы (scrape / health / bootstrap)
|
||||
|
||||
| Канал | Где | Почему это не обход |
|
||||
| --- | --- | --- |
|
||||
| health API | `127.0.0.1:8081` внутри контейнера API, только `/health/live` и `/health/ready` | loopback-only; бизнес-API на нём нет (аудит K: 8081 закрыт из сети) |
|
||||
| health nginx → API | `https://api-N:8080/health/ready` с LB-сертификатом | тот же mTLS, что у бизнес-трафика |
|
||||
| self-метрики Collector | `127.0.0.1:8888`, сбор receiver `prometheus/self` внутри того же процесса | loopback-only; дальше идёт по mTLS-pipeline |
|
||||
| Prometheus scrape | нет ни одного scrape-задания | все метрики приходят OTLP push по mTLS |
|
||||
| healthcheck Loki/Tempo | TCP-connect на 127.0.0.1 | HTTP без клиентского сертификата невозможен |
|
||||
| PostgreSQL local socket | `trust` только для Unix-сокета внутри контейнера | доступен только через `docker compose exec`, то есть оператору |
|
||||
| Erlang distribution | `rabbit@localhost`, epmd и 25672 на 127.0.0.1 | из сети закрыто (аудит K) |
|
||||
| bootstrap | `migrate`, `minio-init` — одноразовые задания с операторскими credentials | завершаются после настройки; у сервисов этих credentials нет |
|
||||
|
||||
## Ограничения модели
|
||||
|
||||
- Collector не связывает resource-атрибуты (`service.name`) с identity сертификата: `api-otel` может отправить данные с `service.name=messenger-worker`. Защищён сам факт ingest (только Ingest CA), но не подделка источника.
|
||||
- В Prometheus, Loki и Tempo нет разграничения прав по identity: любой клиент Telemetry Clients CA и пишет, и читает. Least privilege обеспечен выдачей этого CA только трём identity (collector, grafana, operator).
|
||||
- Отзыв клиентских сертификатов проверяется только в API (serial denylist). RabbitMQ, OpenBao и Collector отзыв не проверяют. Отозвать identity в RabbitMQ — удалить пользователя из `definitions.json`; в OpenBao — удалить роль `auth/cert/certs/<role>`.
|
||||
- Уже открытые соединения повторно не проверяются: отзыв действует на новые соединения (keep-alive nginx → API — 4 с).
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
# Панели dashboard «Messenger — наблюдаемость (lab6)»
|
||||
|
||||
Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).
|
||||
|
||||
## SLO (окно 30 мин) и алерты
|
||||
|
||||
**Доступность (SLO 99%)** (stat, единица: доля (0..1))
|
||||
— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.
|
||||
|
||||
```promql
|
||||
messenger:slo_availability:ratio30m
|
||||
```
|
||||
|
||||
**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1))
|
||||
— Доля запросов в bucket le=0.5 s за 30 мин.
|
||||
|
||||
```promql
|
||||
messenger:slo_latency:ratio30m
|
||||
```
|
||||
|
||||
**Error ratio 5xx (2m)** (stat, единица: доля (0..1))
|
||||
|
||||
```promql
|
||||
messenger:http_error_ratio:rate2m
|
||||
```
|
||||
|
||||
**Старейшее задание** (stat, единица: секунды)
|
||||
|
||||
```promql
|
||||
max(messenger_queue_oldest_age_seconds{queue="attachments.process"})
|
||||
```
|
||||
|
||||
**Алерты Prometheus (firing/pending)** (alertlist)
|
||||
|
||||
Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).
|
||||
|
||||
|
||||
**RPS API** (stat, единица: запросы/с)
|
||||
|
||||
```promql
|
||||
messenger:http_requests:rate2m
|
||||
```
|
||||
|
||||
**p95 API (5m)** (stat, единица: секунды)
|
||||
|
||||
```promql
|
||||
messenger:http_latency_p95:rate5m
|
||||
```
|
||||
|
||||
**Загрузка worker** (stat, единица: доля (0..1))
|
||||
|
||||
```promql
|
||||
messenger:worker_saturation:ratio
|
||||
```
|
||||
|
||||
**Ожидают пул БД** (stat, единица: штуки)
|
||||
|
||||
```promql
|
||||
max(db_client_connection_pending_requests)
|
||||
```
|
||||
|
||||
## API (messenger-api, все реплики)
|
||||
|
||||
**RPS по маршруту и методу** (timeseries, единица: запросы/с)
|
||||
— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.
|
||||
|
||||
```promql
|
||||
sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}}
|
||||
```
|
||||
|
||||
**RPS по классу статуса** (timeseries, единица: запросы/с)
|
||||
|
||||
```promql
|
||||
sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}}
|
||||
```
|
||||
|
||||
**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды)
|
||||
— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Доля ошибок 5xx** (timeseries, единица: доля (0..1))
|
||||
— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.
|
||||
|
||||
```promql
|
||||
(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все
|
||||
```
|
||||
|
||||
**p95 по маршруту** (timeseries, единица: секунды)
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}}
|
||||
```
|
||||
|
||||
**Запросы в обработке** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}}
|
||||
```
|
||||
|
||||
## Очередь и worker
|
||||
|
||||
**Jobs по исходу (в сек.)** (timeseries, единица: операции/с)
|
||||
— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1
|
||||
|
||||
```promql
|
||||
sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}}
|
||||
```
|
||||
|
||||
**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды)
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки)
|
||||
— max() по репликам API: каждая публикует одно и то же значение из БД.
|
||||
|
||||
```promql
|
||||
max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}}
|
||||
```
|
||||
|
||||
**Возраст старейшей job** (timeseries, единица: секунды)
|
||||
— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.
|
||||
|
||||
```promql
|
||||
max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age
|
||||
```
|
||||
|
||||
**Время до ready p50/p95/p99** (timeseries, единица: секунды)
|
||||
— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95
|
||||
```
|
||||
```promql
|
||||
histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99
|
||||
```
|
||||
|
||||
**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum(messenger_worker_busy) # busy
|
||||
```
|
||||
```promql
|
||||
sum(messenger_worker_capacity) # capacity
|
||||
```
|
||||
```promql
|
||||
messenger:worker_saturation:ratio # saturation
|
||||
```
|
||||
|
||||
## Ресурсы: пул БД, CPU, RAM
|
||||
|
||||
**Пул БД: used / idle / waiting** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}}
|
||||
```
|
||||
```promql
|
||||
sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting
|
||||
```
|
||||
```promql
|
||||
sum by (service_name) (db_client_connection_max) # {{service_name}} max
|
||||
```
|
||||
|
||||
**CPU процессов (ядра)** (timeseries, единица: доля (0..1))
|
||||
— worker: self + завершённые дочерние процессы обработки изображений.
|
||||
|
||||
```promql
|
||||
sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}}
|
||||
```
|
||||
```promql
|
||||
rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector
|
||||
```
|
||||
|
||||
**RSS процессов** (timeseries, единица: байты)
|
||||
|
||||
```promql
|
||||
process_memory_usage_bytes # {{service_name}} {{instance}}
|
||||
```
|
||||
```promql
|
||||
otelcol_process_memory_rss_bytes # otel-collector
|
||||
```
|
||||
|
||||
**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки)
|
||||
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points
|
||||
```
|
||||
```promql
|
||||
sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}}
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full)
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs
|
||||
```
|
||||
```promql
|
||||
sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full)
|
||||
```
|
||||
```promql
|
||||
sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}}
|
||||
```
|
||||
|
||||
## Логи и трассы
|
||||
|
||||
**Логи API и worker (warning/error)** (logs)
|
||||
|
||||
```
|
||||
{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"
|
||||
```
|
||||
|
||||
**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table)
|
||||
|
||||
```
|
||||
{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}
|
||||
```
|
||||
@@ -0,0 +1,90 @@
|
||||
# Runbooks алертов (lab6)
|
||||
|
||||
Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana
|
||||
(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой:
|
||||
|
||||
```sh
|
||||
curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool
|
||||
```
|
||||
|
||||
Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved).
|
||||
|
||||
Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в
|
||||
цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID».
|
||||
Все шаги ниже используют только сохранённую telemetry.
|
||||
|
||||
---
|
||||
|
||||
## MessengerApiHighErrorRate
|
||||
|
||||
**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`.
|
||||
Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы).
|
||||
4xx (неверный пароль, чужой чат, 409) в числитель не входят.
|
||||
|
||||
1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента.
|
||||
2. Prometheus: на каком маршруте ошибки
|
||||
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`.
|
||||
3. Отличить БД от handler:
|
||||
- пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`.
|
||||
Ожидание пула > 0 → соединения заняты долгими запросами к БД;
|
||||
- Tempo (Explore → Tempo → TraceQL):
|
||||
`{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе
|
||||
db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением
|
||||
`LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность);
|
||||
- если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler
|
||||
(`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину).
|
||||
4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error`
|
||||
(500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`.
|
||||
По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= "<request_id>"` → поле `trace_id`
|
||||
→ кнопка «Open trace».
|
||||
5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state,
|
||||
wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` —
|
||||
найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend(<pid>)`.
|
||||
6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин).
|
||||
|
||||
## MessengerQueueLagGrowing
|
||||
|
||||
**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`.
|
||||
Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API,
|
||||
поэтому `max()`.
|
||||
|
||||
1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker».
|
||||
- `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает;
|
||||
- `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности);
|
||||
- много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`.
|
||||
2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id="<id>"}`. Если в трассе есть
|
||||
PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал.
|
||||
Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог
|
||||
`outbox_relay_paused` (брокер недоступен).
|
||||
3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие
|
||||
`job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry.
|
||||
4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование
|
||||
`docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди.
|
||||
5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с.
|
||||
|
||||
## MessengerApiLatencyHigh
|
||||
|
||||
**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3:
|
||||
по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время
|
||||
(db / S3 / handler).
|
||||
|
||||
## MessengerDbPoolWaiting
|
||||
|
||||
**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или
|
||||
блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`).
|
||||
|
||||
## MessengerWorkerSaturated
|
||||
|
||||
**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker;
|
||||
проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее).
|
||||
|
||||
## TelemetryExportFailing
|
||||
|
||||
**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`.
|
||||
Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди.
|
||||
|
||||
1. `docker compose ps tempo loki prometheus` — какой backend недоступен.
|
||||
2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и
|
||||
`otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено.
|
||||
3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются —
|
||||
их объём виден в счётчиках и фиксируется в отчёте инцидента.
|
||||
@@ -0,0 +1,11 @@
|
||||
10:06:50 PASS acl_outsider eve: 404 on chat/history/metadata/content/variant/post, cannot attach a foreign file
|
||||
10:06:50 PASS identity_spoofing forged X-User-Id ignored; without Bearer 401; Basic on business endpoint 401
|
||||
10:06:50 PASS logout revoked token: 401 on ['api-1', 'api-2']
|
||||
10:06:50 session expires_at moved to the past in PostgreSQL; the Valkey cache entry lives at most 30 s
|
||||
10:07:22 PASS expiry expired session: 401 on ['api-1', 'api-2'] after the 30 s cache bound
|
||||
10:07:23 PASS rehash stored m=8192,t=1,p=1 -> m=19456,t=2,p=1 after a successful login
|
||||
10:07:23 PASS size_limits JSON 70 kB -> 413 (payload_too_large), upload > 10 MiB -> 413
|
||||
10:07:23 PASS rate_limit_user 5 failures on ['api-1', 'api-2'] -> 429 Retry-After=60s even with the right password
|
||||
10:07:24 PASS rate_limit_ip per-IP limit: 429 after 107 attempts from one address
|
||||
10:07:24 waiting for the rate-limit window to expire so that later tests are not affected
|
||||
PASS: acl_outsider, identity_spoofing, logout, expiry, rehash, size_limits, rate_limit_user, rate_limit_ip
|
||||
@@ -0,0 +1,8 @@
|
||||
11:08:38 openbao: sealed
|
||||
11:08:40 sealed: running services keep working -> PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
11:09:00 refresher: openbao_unavailable ... keeping cached secret version (seen in API/worker logs)
|
||||
11:09:35 worker-2 restarted while sealed: health=starting, waiting for OpenBao=True
|
||||
11:09:36 upload during the seal processed by the remaining worker: ready
|
||||
11:09:37 openbao: unsealed with 2 of 3 operator shares
|
||||
11:09:52 worker-2 healthy 16 s after unseal (sealed for 58 s in total)
|
||||
PASS: sealed OpenBao -> running services continue, new process waits, recovery after unseal
|
||||
@@ -0,0 +1,11 @@
|
||||
10:12:23 current lb-client serial 0E30F0061EA3866362334D7C34B7F93B9BD61418
|
||||
10:12:24 issued lb-client (lb.client.messenger) serial=0E30F0061EA3866362334D7C34B7F93B9BD6142A valid 90d
|
||||
10:12:25 nginx reloaded with the new client certificate (1.4 s after issue)
|
||||
10:12:25 before revocation: old=HTTP/1.1 200 OK new=HTTP/1.1 200 OK
|
||||
10:12:25 revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418
|
||||
10:12:28 api-1 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response
|
||||
10:12:29 api-2 after revocation: old -> closed without response | new -> HTTP/1.1 200 OK | expired -> closed without response
|
||||
10:12:29 API log: mtls_client_rejected reason=revoked serial=0E30F0061EA3866362334D7C34B7F93B9BD61418
|
||||
10:12:34 traffic during rotation: 161 requests, 161 ok, unexpected errors 0
|
||||
10:12:35 PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
PASS: lb-client rotated (0E30F0061EA3866362334D7C34B7F93B9BD61418 -> 0E30F0061EA3866362334D7C34B7F93B9BD6142A) without image rebuild, old serial revoked, expired cert refused
|
||||
@@ -0,0 +1,92 @@
|
||||
ok 1 series Доступность (SLO 99%) Доступность (SLO 99%)
|
||||
ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%)
|
||||
ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m)
|
||||
ok 1 series Старейшее задание Старейшее задание
|
||||
ok 1 series RPS API RPS API
|
||||
ok 1 series p95 API (5m) p95 API (5m)
|
||||
ok 1 series Загрузка worker Загрузка worker
|
||||
ok 1 series Ожидают пул БД Ожидают пул БД
|
||||
ok 12 series RPS по маршруту и методу {{http_request_method}} {{http_route}}
|
||||
ok 3 series RPS по классу статуса {{status_class}}
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95
|
||||
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99
|
||||
ok 1 series Доля ошибок 5xx 5xx / все
|
||||
ok 10 series p95 по маршруту {{http_route}}
|
||||
ok 2 series Запросы в обработке {{instance}}
|
||||
ok 6 series Jobs по исходу (в сек.) {{outcome}}
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p50
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p95
|
||||
ok 1 series Длительность job (succeeded) p50/p95/p99 p99
|
||||
ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}}
|
||||
ok 1 series Возраст старейшей job oldest age
|
||||
ok 1 series Время до ready p50/p95/p99 p50
|
||||
ok 1 series Время до ready p50/p95/p99 p95
|
||||
ok 1 series Время до ready p50/p95/p99 p99
|
||||
ok 1 series Насыщение worker: занято / ёмкость busy
|
||||
ok 1 series Насыщение worker: занято / ёмкость capacity
|
||||
ok 1 series Насыщение worker: занято / ёмкость saturation
|
||||
ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}}
|
||||
ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting
|
||||
ok 2 series Пул БД: used / idle / waiting {{service_name}} max
|
||||
ok 4 series CPU процессов (ядра) {{service_name}} {{instance}}
|
||||
ok 1 series CPU процессов (ядра) otel-collector
|
||||
ok 4 series RSS процессов {{service_name}} {{instance}}
|
||||
ok 1 series RSS процессов otel-collector
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points
|
||||
ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}}
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full)
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs
|
||||
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full)
|
||||
ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}}
|
||||
panels: 32, prometheus queries without data: 0
|
||||
grafana prometheus ok 84 values Доступность (SLO 99%)
|
||||
grafana prometheus ok 84 values Латентность ≤500 ms (SLO 95%)
|
||||
grafana prometheus ok 84 values Error ratio 5xx (2m)
|
||||
grafana prometheus ok 86 values Старейшее задание
|
||||
grafana prometheus ok 84 values RPS API
|
||||
grafana prometheus ok 84 values p95 API (5m)
|
||||
grafana prometheus ok 86 values Загрузка worker
|
||||
grafana prometheus ok 86 values Ожидают пул БД
|
||||
grafana prometheus ok 789 values RPS по маршруту и методу
|
||||
grafana prometheus ok 179 values RPS по классу статуса
|
||||
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 82 values Латентность API p50/p95/p99 (из histogram bucket
|
||||
grafana prometheus ok 82 values Доля ошибок 5xx
|
||||
grafana prometheus ok 683 values p95 по маршруту
|
||||
grafana prometheus ok 170 values Запросы в обработке
|
||||
grafana prometheus ok 273 values Jobs по исходу (в сек.)
|
||||
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 77 values Длительность job (succeeded) p50/p95/p99
|
||||
grafana prometheus ok 258 values Глубина очереди (из PostgreSQL)
|
||||
grafana prometheus ok 86 values Возраст старейшей job
|
||||
grafana prometheus ok 77 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 77 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 77 values Время до ready p50/p95/p99
|
||||
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 86 values Насыщение worker: занято / ёмкость
|
||||
grafana prometheus ok 344 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 172 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 172 values Пул БД: used / idle / waiting
|
||||
grafana prometheus ok 350 values CPU процессов (ядра)
|
||||
grafana prometheus ok 91 values CPU процессов (ядра)
|
||||
grafana prometheus ok 402 values RSS процессов
|
||||
grafana prometheus ok 105 values RSS процессов
|
||||
grafana prometheus ok 76 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 80 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 247 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana prometheus ok 273 values Telemetry pipeline: принято / отправлено / ошибк
|
||||
grafana loki ok 100 values Логи API и worker (warning/error)
|
||||
grafana tempo ok 20 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy)
|
||||
via Grafana: queries without data or with errors: 0
|
||||
@@ -0,0 +1,13 @@
|
||||
# 2026-11-11T08:10:20Z
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.265}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab7-api-1-1 Up 38 seconds (healthy)
|
||||
$ docker compose start api-1 api-2
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.672}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab7-api-1-1 Up About a minute (healthy)
|
||||
$ docker compose start api-1 api-2
|
||||
@@ -0,0 +1,342 @@
|
||||
[
|
||||
{
|
||||
"container": "api-1-1",
|
||||
"user": "10001:10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.5,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "api-2-1",
|
||||
"user": "10001:10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.5,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "grafana-1",
|
||||
"user": "472",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.5,
|
||||
"memory_mib": 256,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [
|
||||
"127.0.0.1:3000->3000/tcp"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "loki-1",
|
||||
"user": "10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [
|
||||
"127.0.0.1:3100->3100/tcp"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "minio-1",
|
||||
"user": "65532",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.5,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "nginx-1",
|
||||
"user": "101:101",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/etc/nginx/upstream",
|
||||
"/tmp",
|
||||
"/var/cache/nginx",
|
||||
"/var/run"
|
||||
],
|
||||
"cpus": 0.25,
|
||||
"memory_mib": 64,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [
|
||||
"127.0.0.1:8443->8443/tcp"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "openbao-1",
|
||||
"user": "100:1000",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.5,
|
||||
"memory_mib": 256,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "otel-collector-1",
|
||||
"user": "10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 256,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "postgres-1",
|
||||
"user": "70:70",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp",
|
||||
"/var/run/postgresql"
|
||||
],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "prometheus-1",
|
||||
"user": "nobody",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [
|
||||
"127.0.0.1:9090->9090/tcp"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "rabbitmq-1",
|
||||
"user": "100:101",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/etc/rabbitmq/conf.d",
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 512,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "tempo-1",
|
||||
"user": "10001:10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 1.0,
|
||||
"memory_mib": 1024,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [
|
||||
"127.0.0.1:3200->3200/tcp"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "valkey-1",
|
||||
"user": "999:1000",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/data",
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 0.25,
|
||||
"memory_mib": 128,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "worker-1",
|
||||
"user": "10001:10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 2.0,
|
||||
"memory_mib": 1536,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"container": "worker-2",
|
||||
"user": "10001:10001",
|
||||
"privileged": false,
|
||||
"network": "mtusi-lab7_default",
|
||||
"pid": "private",
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": null,
|
||||
"no_new_privileges": true,
|
||||
"read_only_rootfs": true,
|
||||
"tmpfs": [
|
||||
"/tmp"
|
||||
],
|
||||
"cpus": 2.0,
|
||||
"memory_mib": 1536,
|
||||
"pids_limit": 256,
|
||||
"docker_socket": false,
|
||||
"published": [],
|
||||
"problems": []
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
container user capDrop nnp ro cpus mem/pids published problems
|
||||
api-1-1 10001:10001 ALL True True 0.5 512/256 - ok
|
||||
api-2-1 10001:10001 ALL True True 0.5 512/256 - ok
|
||||
grafana-1 472 ALL True True 0.5 256/256 127.0.0.1:3000 ok
|
||||
loki-1 10001 ALL True True 1.0 512/256 127.0.0.1:3100 ok
|
||||
minio-1 65532 ALL True True 0.5 512/256 - ok
|
||||
nginx-1 101:101 ALL True True 0.25 64/256 127.0.0.1:8443 ok
|
||||
openbao-1 100:1000 ALL True True 0.5 256/256 - ok
|
||||
otel-collector-1 10001 ALL True True 1.0 256/256 - ok
|
||||
postgres-1 70:70 ALL True True 1.0 512/256 - ok
|
||||
prometheus-1 nobody ALL True True 1.0 512/256 127.0.0.1:9090 ok
|
||||
rabbitmq-1 100:101 ALL True True 1.0 512/256 - ok
|
||||
tempo-1 10001:10001 ALL True True 1.0 1024/256 127.0.0.1:3200 ok
|
||||
valkey-1 999:1000 ALL True True 0.25 128/256 - ok
|
||||
worker-1 10001:10001 ALL True True 2.0 1536/256 - ok
|
||||
worker-2 10001:10001 ALL True True 2.0 1536/256 - ok
|
||||
published ports allowed: {'8443': 'nginx business ingress (TLS)', '3000': 'Grafana (HTTPS + login)', '9090': 'Prometheus (mTLS operator)', '3100': 'Loki (mTLS operator)', '3200': 'Tempo (mTLS operator)'}
|
||||
15 containers, 0 problems
|
||||
@@ -0,0 +1,16 @@
|
||||
Trivy 0.58.1, DB 2026-11-11; scanned with docker save (no Docker socket in the scanner)
|
||||
|
||||
== app BEFORE (python 3.12.8, pillow 11.0.0, urllib3 2.3.0)
|
||||
/scan/app.tar (debian 12.9) os-pkgs {'CRITICAL': 9, 'HIGH': 145, 'LOW': 158, 'MEDIUM': 217} | HIGH+CRIT with fix: 56/154 | secrets: 0
|
||||
Python lang-pkgs {'CRITICAL': 5, 'HIGH': 43, 'LOW': 2, 'MEDIUM': 26} | HIGH+CRIT with fix: 48/48 | secrets: 0
|
||||
|
||||
== app AFTER (python 3.12.16, pillow 13.0.0, urllib3 2.8.0)
|
||||
/scan/app.tar (debian 12.15) os-pkgs {'CRITICAL': 2, 'HIGH': 96, 'LOW': 108, 'MEDIUM': 129, 'UNKNOWN': 10} | HIGH+CRIT with fix: 0/98 | secrets: 0
|
||||
Python lang-pkgs {'HIGH': 3, 'LOW': 2, 'MEDIUM': 9} | HIGH+CRIT with fix: 3/3 | secrets: 0
|
||||
|
||||
== nginx:1.27.3-alpine
|
||||
/scan/nginx.tar (alpine 3.20.5) os-pkgs {'CRITICAL': 3, 'HIGH': 37, 'LOW': 31, 'MEDIUM': 33} | HIGH+CRIT with fix: 40/40 | secrets: 0
|
||||
|
||||
== openbao/openbao:2.1.0
|
||||
/scan/openbao.tar (alpine 3.20.3) os-pkgs {'CRITICAL': 2, 'HIGH': 26, 'LOW': 23, 'MEDIUM': 14} | HIGH+CRIT with fix: 28/28 | secrets: 0
|
||||
bin/bao lang-pkgs {'CRITICAL': 2, 'HIGH': 61, 'LOW': 9, 'MEDIUM': 68, 'UNKNOWN': 2} | HIGH+CRIT with fix: 59/63 | secrets: 0
|
||||
@@ -0,0 +1,80 @@
|
||||
10:18:35 scenario A: scenario_a
|
||||
10:18:35 client saw: missing X-Request-Id
|
||||
10:18:35 committed attachment a3900cbb-c50d-496c-9962-ba0ec39928d3 survived the API crash
|
||||
10:18:35 statuses: processing -> ready
|
||||
10:18:35 audit attempt=1 enqueued api:api-1
|
||||
10:18:35 audit attempt=1 claimed 2e84bfa8176e:4f81b283
|
||||
10:18:35 audit attempt=1 succeeded 2e84bfa8176e:4f81b283
|
||||
10:18:36 scenario A PASS (1.9s)
|
||||
10:18:36 scenario B: scenario_b
|
||||
10:18:59 statuses: queued -> processing -> queued -> ready
|
||||
10:18:59 audit attempt=1 enqueued api:api-2
|
||||
10:18:59 audit attempt=1 claimed eb9974d4233a:3bafe435
|
||||
10:18:59 audit attempt=1 retry_scheduled sweeper:2e84bfa8176e:4f81b28 lease_expired delay=1.34s
|
||||
10:18:59 audit attempt=2 claimed 2e84bfa8176e:4f81b283
|
||||
10:18:59 audit attempt=2 succeeded 2e84bfa8176e:4f81b283
|
||||
10:19:00 scenario B PASS (23.6s)
|
||||
10:19:00 scenario C: scenario_c
|
||||
10:19:01 statuses: queued -> ready
|
||||
10:19:01 audit attempt=1 enqueued api:api-2
|
||||
10:19:01 audit attempt=1 claimed eb9974d4233a:f07c7c15
|
||||
10:19:01 audit attempt=1 succeeded eb9974d4233a:f07c7c15
|
||||
10:19:11 scenario C PASS (10.8s)
|
||||
10:19:11 scenario D: scenario_d
|
||||
10:19:22 statuses: processing -> queued -> ready
|
||||
10:19:22 audit attempt=1 enqueued api:api-1
|
||||
10:19:22 audit attempt=1 claimed 2e84bfa8176e:4f81b283
|
||||
10:19:22 audit attempt=1 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=1.33s
|
||||
10:19:22 audit attempt=2 claimed eb9974d4233a:ea4f119d
|
||||
10:19:22 audit attempt=2 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=2.5s
|
||||
10:19:22 audit attempt=3 claimed 2e84bfa8176e:4f81b283
|
||||
10:19:22 audit attempt=3 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=5.78s
|
||||
10:19:22 audit attempt=4 claimed eb9974d4233a:ea4f119d
|
||||
10:19:22 audit attempt=4 succeeded eb9974d4233a:ea4f119d
|
||||
10:19:24 backoff delays: [1.33, 2.5, 5.78]
|
||||
10:19:24 scenario D PASS (12.6s)
|
||||
10:19:24 scenario E: scenario_e
|
||||
10:19:24 audit attempt=1 enqueued api:api-2
|
||||
10:19:24 audit attempt=1 claimed 2e84bfa8176e:4f81b283
|
||||
10:19:24 audit attempt=1 failed 2e84bfa8176e:4f81b283 invalid_image
|
||||
10:19:24 scenario E PASS (0.7s)
|
||||
10:19:24 scenario F: scenario_f
|
||||
10:19:48 failed registry contains the job; replay
|
||||
10:20:11 statuses: queued -> ready
|
||||
10:20:11 audit attempt=1 enqueued api:api-1
|
||||
10:20:11 audit attempt=1 claimed eb9974d4233a:ea4f119d
|
||||
10:20:11 audit attempt=1 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=1.28s
|
||||
10:20:11 audit attempt=2 claimed 2e84bfa8176e:4f81b283
|
||||
10:20:11 audit attempt=2 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=2.55s
|
||||
10:20:11 audit attempt=3 claimed eb9974d4233a:ea4f119d
|
||||
10:20:11 audit attempt=3 retry_scheduled eb9974d4233a:ea4f119d dependency_unavailable delay=6.47s
|
||||
10:20:11 audit attempt=4 claimed 2e84bfa8176e:4f81b283
|
||||
10:20:11 audit attempt=4 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=9.94s
|
||||
10:20:11 audit attempt=5 claimed eb9974d4233a:ea4f119d
|
||||
10:20:11 audit attempt=5 failed eb9974d4233a:ea4f119d retry_exhausted
|
||||
10:20:11 audit attempt=6 replayed operator:test-suite
|
||||
10:20:11 audit attempt=6 claimed 2e84bfa8176e:4f81b283
|
||||
10:20:11 audit attempt=6 retry_scheduled 2e84bfa8176e:4f81b283 dependency_unavailable delay=20.59s
|
||||
10:20:11 audit attempt=7 claimed eb9974d4233a:ea4f119d
|
||||
10:20:11 audit attempt=7 succeeded eb9974d4233a:ea4f119d
|
||||
10:20:13 second replay refused: job is succeeded, attachment is ready
|
||||
10:20:14 scenario F PASS (50.2s)
|
||||
10:20:14 scenario G: scenario_g
|
||||
10:20:38 statuses: processing -> queued -> processing -> ready
|
||||
10:20:38 audit attempt=1 enqueued api:api-1
|
||||
10:20:38 audit attempt=1 claimed 2e84bfa8176e:4f81b283
|
||||
10:20:38 audit attempt=1 retry_scheduled sweeper:eb9974d4233a:ea4f119 lease_expired delay=1.12s
|
||||
10:20:38 audit attempt=2 claimed eb9974d4233a:ea4f119d
|
||||
10:20:38 audit attempt=2 succeeded eb9974d4233a:ea4f119d
|
||||
10:21:11 scenario G PASS (56.4s)
|
||||
10:21:11 scenario H: scenario_h
|
||||
10:21:15 DLQ depth 0 -> 1
|
||||
10:21:15 scenario H PASS (4.1s)
|
||||
10:21:15 scenario I: scenario_i
|
||||
10:21:19 broker down: upload 202, outbox holds 1 unpublished job
|
||||
10:21:25 statuses: queued -> ready
|
||||
10:21:25 audit attempt=1 enqueued api:api-1
|
||||
10:21:25 audit attempt=1 claimed 2e84bfa8176e:4f81b283
|
||||
10:21:25 audit attempt=1 succeeded 2e84bfa8176e:4f81b283
|
||||
10:21:26 scenario I PASS (11.4s)
|
||||
PASS: failure windows A B C D E F G H I
|
||||
@@ -0,0 +1,14 @@
|
||||
10:18:08 stop all workers
|
||||
10:18:08 upload 202 queued id=59f4941d-1a35-4d70-a976-c8ed89746f38
|
||||
10:18:08 message 0f104c92-c0dd-46a0-b906-f58c9fde4017 created with queued attachment
|
||||
10:18:10 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0}
|
||||
10:18:11 restart api-1 api-2 rabbitmq (volumes kept)
|
||||
10:18:23 started before: mtusi-lab7-api-1-1 2026-11-11T07:17:26.446583425Z; mtusi-lab7-api-2-1 2026-11-11T07:17:38.07347875Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:17:27.208089551Z
|
||||
10:18:23 started after: mtusi-lab7-api-1-1 2026-11-11T07:18:11.610777668Z; mtusi-lab7-api-2-1 2026-11-11T07:18:11.669354168Z; mtusi-lab7-rabbitmq-1 2026-11-11T07:18:12.345023127Z
|
||||
10:18:24 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0}
|
||||
10:18:24 start workers
|
||||
10:18:27 status sequence observed: queued -> ready
|
||||
10:18:28 objects: ['attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/original', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/crop.jpg', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/optimized.webp', 'attachments/59f4941d-1a35-4d70-a976-c8ed89746f38/v1/thumbnail.jpg']
|
||||
10:18:28 redelivery: republish the finished job through the outbox (duplicate message)
|
||||
10:18:34 duplicate acked as no-op: variants=3 successes=1
|
||||
PASS: worker down / API+broker restart / redelivery no-op
|
||||
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"argon2_bench_api_container_0_5_cpu": [
|
||||
{
|
||||
"m_kib": 19456,
|
||||
"t": 2,
|
||||
"p": 1,
|
||||
"p50_ms": 11.3,
|
||||
"max_ms": 66.6,
|
||||
"peak_rss_mib": 37.3
|
||||
},
|
||||
{
|
||||
"m_kib": 12288,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"p50_ms": 9.3,
|
||||
"max_ms": 62.0,
|
||||
"peak_rss_mib": 37.3
|
||||
},
|
||||
{
|
||||
"m_kib": 47104,
|
||||
"t": 1,
|
||||
"p": 1,
|
||||
"p50_ms": 16.6,
|
||||
"max_ms": 69.2,
|
||||
"peak_rss_mib": 83.0
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 3,
|
||||
"p": 1,
|
||||
"p50_ms": 171.6,
|
||||
"max_ms": 175.5,
|
||||
"peak_rss_mib": 101.0
|
||||
},
|
||||
{
|
||||
"m_kib": 65536,
|
||||
"t": 3,
|
||||
"p": 4,
|
||||
"p50_ms": 390.0,
|
||||
"max_ms": 806.1,
|
||||
"peak_rss_mib": 101.2
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"attackers": 0,
|
||||
"seconds": 35,
|
||||
"login_attempts_per_s": 0.0,
|
||||
"login_status_counts": {},
|
||||
"login_401_p50_ms": null,
|
||||
"login_401_p99_ms": null,
|
||||
"probe_ok": 165,
|
||||
"probe_errors": 0,
|
||||
"probe_p50_ms": 9.5,
|
||||
"probe_p99_ms": 46.4,
|
||||
"api_cpu_pct_of_one_core_both_replicas": 10.3,
|
||||
"api_rss_samples": {
|
||||
"mtusi-lab7-api-2-1": "97.59MiB",
|
||||
"mtusi-lab7-api-1-1": "97.34MiB"
|
||||
}
|
||||
},
|
||||
"flood": {
|
||||
"attackers": 32,
|
||||
"seconds": 37,
|
||||
"login_attempts_per_s": 40.4,
|
||||
"login_status_counts": {
|
||||
"401": 1496
|
||||
},
|
||||
"login_401_p50_ms": 802.6,
|
||||
"login_401_p99_ms": 2396.6,
|
||||
"probe_ok": 128,
|
||||
"probe_errors": 0,
|
||||
"probe_p50_ms": 28.5,
|
||||
"probe_p99_ms": 295.0,
|
||||
"api_cpu_pct_of_one_core_both_replicas": 89.6,
|
||||
"api_rss_samples": {
|
||||
"mtusi-lab7-api-2-1": "196.7MiB",
|
||||
"mtusi-lab7-api-1-1": "183.3MiB"
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"PASSWORD_HASH_CONCURRENCY_per_replica": 4,
|
||||
"PASSWORD_HASH_QUEUE_SECONDS": 3,
|
||||
"replicas": 2,
|
||||
"api_cpu_limit": 0.5,
|
||||
"api_mem_limit_mib": 512
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
10:32:44 message sent request_id=94e97b26265542fb9d208a905b5d5701 (text marker kept private)
|
||||
10:32:47 upload 913f8afd-4a07-4a58-b943-ea623e6ec0d0: queued -> queued -> processing -> ready
|
||||
10:32:51 Tempo: trace 715b953a7f7536b44b99c48205a5c639, 42 spans; tree (DB client spans omitted):
|
||||
api SERVER POST /api/v1/chats/{chat_id}/attachments 15.3 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "95e2a7eb2f9e46319830eca2e087ccfb", "messenger.instance_id": "api-2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
|
||||
api CLIENT S3.PutObject 8.4 ms UNSET {"rpc.method": "PutObject"}
|
||||
api PRODUCER attachments.process publish 3.1 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "1", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
|
||||
worker CONSUMER attachments.process process 8.3 ms ERROR {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"}
|
||||
api PRODUCER attachments.process publish 4.8 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.job.attempt": "2", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0"}
|
||||
worker CONSUMER attachments.process process 174.7 ms UNSET {"messaging.message.id": "52bd90b1-34f0-4737-86e7-50e164191b3f", "messenger.attachment_id": "913f8afd-4a07-4a58-b943-ea623e6ec0d0", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"}
|
||||
worker CLIENT S3.GetObject 15.2 ms UNSET {"rpc.method": "GetObject"}
|
||||
worker INTERNAL imageproc 128.0 ms UNSET {"messenger.pipeline_version": "1"}
|
||||
worker CLIENT S3.CopyObject 5.3 ms UNSET {"rpc.method": "CopyObject"}
|
||||
worker CLIENT S3.PutObject 3.0 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.PutObject 2.1 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
|
||||
worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"}
|
||||
10:32:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links
|
||||
10:32:51 Loki: 4 records with trace_id=715b953a7f7536b44b99c48205a5c639: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished')]
|
||||
10:32:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts']
|
||||
10:32:55 HTTP 500 X-Request-Id=2df37bc19c3d4676804688b4abcb2ae1 -> Loki event=unhandled_error trace_id=f63428fc5f86c4f25be261c5db8ecc08
|
||||
10:32:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True
|
||||
10:32:55 message POST trace 3157b85474b226259a91fb7e6ddeec5a: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys']
|
||||
10:32:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs
|
||||
10:32:55 PASS: 1921 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_method', 'http_request_method', 'http_response_status_code', 'http_route', 'http_scheme', 'http_status_code', 'instance', 'job', 'job_type', 'le', 'net_host_name', 'net_host_port', 'net_protocol_name', 'net_protocol_version', 'network_protocol_version', 'otel_scope_name', 'otel_scope_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme']
|
||||
10:32:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me']
|
||||
10:33:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size
|
||||
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
|
||||
@@ -0,0 +1,36 @@
|
||||
# 2026-11-11T08:09:52Z scripts/restart.sh
|
||||
$ docker compose ps (before)
|
||||
mtusi-lab7-api-1-1 Up 51 minutes (healthy)
|
||||
mtusi-lab7-api-2-1 Up 51 minutes (healthy)
|
||||
mtusi-lab7-grafana-1 Up About an hour (healthy)
|
||||
mtusi-lab7-loki-1 Up About an hour (healthy)
|
||||
mtusi-lab7-minio-1 Up About an hour (healthy)
|
||||
mtusi-lab7-nginx-1 Up 43 minutes (healthy)
|
||||
mtusi-lab7-openbao-1 Up About a minute (healthy)
|
||||
mtusi-lab7-otel-collector-1 Up 42 minutes
|
||||
mtusi-lab7-postgres-1 Up About an hour (healthy)
|
||||
mtusi-lab7-prometheus-1 Up About an hour (healthy)
|
||||
mtusi-lab7-rabbitmq-1 Up 40 minutes (healthy)
|
||||
mtusi-lab7-tempo-1 Up About an hour (healthy)
|
||||
mtusi-lab7-valkey-1 Up About an hour (healthy)
|
||||
mtusi-lab7-worker-1 Up 50 minutes (healthy)
|
||||
mtusi-lab7-worker-2 Up 51 seconds (healthy)
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ docker compose ps (after)
|
||||
mtusi-lab7-api-1-1 Up 2 seconds (health: starting)
|
||||
mtusi-lab7-api-2-1 Up 1 second (health: starting)
|
||||
mtusi-lab7-grafana-1 Up 4 seconds (health: starting)
|
||||
mtusi-lab7-loki-1 Up 2 seconds (health: starting)
|
||||
mtusi-lab7-minio-1 Up 7 seconds (healthy)
|
||||
mtusi-lab7-nginx-1 Up 2 seconds (health: starting)
|
||||
mtusi-lab7-openbao-1 Up 7 seconds (healthy)
|
||||
mtusi-lab7-otel-collector-1 Up 4 seconds
|
||||
mtusi-lab7-postgres-1 Up 7 seconds (healthy)
|
||||
mtusi-lab7-prometheus-1 Up 4 seconds (health: starting)
|
||||
mtusi-lab7-rabbitmq-1 Up 6 seconds (healthy)
|
||||
mtusi-lab7-tempo-1 Up 4 seconds (health: starting)
|
||||
mtusi-lab7-valkey-1 Up 7 seconds (healthy)
|
||||
mtusi-lab7-worker-1 Up 1 second (health: starting)
|
||||
mtusi-lab7-worker-2 Up 1 second (health: starting)
|
||||
@@ -0,0 +1,8 @@
|
||||
11:08:20 before: old password accepted, 2 app_worker connections open
|
||||
11:08:21 rotated: app_worker password changed in PostgreSQL and OpenBao (messenger/worker, new KV version)
|
||||
11:08:30 both workers loaded the new KV version and switched the pool (10 s after rotation)
|
||||
11:08:30 terminated all existing app_worker sessions: the pools must reconnect with the new password
|
||||
11:08:36 3 uploads after the forced reconnect are ready; 2 app_worker connections open again
|
||||
11:08:37 probe: old password -> rejected: FATAL: password authentication failed for user "app_worker" | new password -> accepted
|
||||
11:08:37 traffic: 116 requests, unexpected errors 0, uploads 20; failed jobs since rotation: 0
|
||||
PASS: app_worker password rotated via OpenBao without rebuild/restart; old password revoked
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,9 @@
|
||||
# 2026-11-11T08:08:18Z (OpenBao sealed at this moment: running services use the secrets they already hold)
|
||||
$ make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
$ make validate
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
$ make test CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
$ make test-security CA_FILE=/Users/alenchha/mtusi2026lab7/secrets/pki/ca.crt
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,124 @@
|
||||
PASS A LB -> api-1:8080 no client cert expect=reject rejected: closed without response
|
||||
PASS A LB -> api-1:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
|
||||
PASS A LB -> api-1:8080 same CA, identity intruder expect=reject rejected: closed without response
|
||||
PASS A LB -> api-1:8080 same CA, identity worker expect=reject rejected: closed without response
|
||||
PASS A LB -> api-1:8080 same CA, identity api expect=reject rejected: closed without response
|
||||
PASS A LB -> api-1:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS A LB -> api-1:8080 plaintext client expect=reject server answered b''
|
||||
PASS A LB -> api-2:8080 no client cert expect=reject rejected: closed without response
|
||||
PASS A LB -> api-2:8080 foreign CA (claims lb identity) expect=reject rejected: closed without response
|
||||
PASS A LB -> api-2:8080 same CA, identity intruder expect=reject rejected: closed without response
|
||||
PASS A LB -> api-2:8080 same CA, identity worker expect=reject rejected: closed without response
|
||||
PASS A LB -> api-2:8080 same CA, identity api expect=reject rejected: closed without response
|
||||
PASS A LB -> api-2:8080 lb.client.messenger expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS A LB -> api-2:8080 plaintext client expect=reject server answered b''
|
||||
PASS I client -> nginx:8443 trusted CA, hostname nginx expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS I client -> nginx:8443 untrusted CA in client trust store expect=reject rejected: server cert verify failed: unable to get local issuer certificate
|
||||
PASS I client -> nginx:8443 wrong hostname (evil.example) expect=reject rejected: server cert verify failed: Hostname mismatch, certificate is not valid for 'evil.example'.
|
||||
PASS I client -> nginx:8443 plaintext client expect=reject server answered b'HTTP/1.1 400 Bad Request'
|
||||
PASS B svc -> postgres:5432 plaintext (sslmode=disable) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: pg_hba.c
|
||||
PASS B svc -> postgres:5432 foreign CA as root (verify-full) expect=reject OperationalError: connection failed: connection to server at "172.22.0.10", port 5432 failed: SSL error: certi
|
||||
PASS B svc -> postgres:5432 hostname mismatch (IP, verify-full) expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: server certificate for "postgres"
|
||||
PASS B svc -> postgres:5432 TLS verify-full + app_api password expect=accept connected
|
||||
PASS B svc -> postgres:5432 wrong password expect=reject connection failed: connection to server at "172.22.0.10", port 5432 failed: FATAL: password authentication fa
|
||||
PASS B app_worker privileges SELECT count(*) FROM users expect=deny InsufficientPrivilege
|
||||
PASS B app_worker privileges SELECT count(*) FROM messages expect=deny InsufficientPrivilege
|
||||
PASS B app_worker privileges SELECT count(*) FROM sessions expect=deny InsufficientPrivilege
|
||||
PASS B app_worker privileges DELETE FROM attachments WHERE false expect=deny InsufficientPrivilege
|
||||
PASS B app_worker privileges SELECT count(*) FROM attachment_jobs expect=allow executed
|
||||
PASS B app_api privileges SELECT count(*) FROM users expect=allow executed
|
||||
PASS B app_api privileges CREATE TABLE audit_probe (x int) expect=deny InsufficientPrivilege
|
||||
PASS B app_api privileges ALTER ROLE app_worker PASSWORD 'x' expect=deny InsufficientPrivilege
|
||||
PASS B app_api privileges SELECT count(*) FROM schema_migrations expect=deny InsufficientPrivilege
|
||||
PASS C svc -> minio:9000 foreign CA as trust anchor expect=reject SSLError
|
||||
PASS C svc -> minio:9000 plaintext http:// expect=reject ConnectionClosedError
|
||||
PASS C worker S3 rights PUT attachments/* expect=allow ok
|
||||
PASS C worker S3 rights GET attachments/* expect=deny AccessDenied
|
||||
PASS C worker S3 rights DELETE attachments/* expect=deny AccessDenied
|
||||
PASS C worker S3 rights PUT staging/* expect=deny AccessDenied
|
||||
PASS C worker S3 rights LIST bucket expect=deny AccessDenied
|
||||
PASS C worker S3 rights PUT outside prefixes expect=deny AccessDenied
|
||||
PASS C api S3 rights PUT staging/* expect=allow ok
|
||||
PASS C api S3 rights PUT attachments/* (only worker publishes) expect=deny AccessDenied
|
||||
PASS C api S3 rights GET attachments/* expect=allow ok
|
||||
PASS C api S3 rights LIST bucket (reconciler) expect=allow ok
|
||||
PASS C api S3 rights create bucket expect=deny AccessDenied
|
||||
PASS C api S3 rights cleanup audit objects expect=allow ok
|
||||
PASS D svc -> rabbitmq:5672 plaintext client expect=reject connection failed: ConnectionRefusedError
|
||||
PASS D svc -> rabbitmq:5671 no client cert expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV13_ALERT_
|
||||
PASS D svc -> rabbitmq:5671 foreign CA (claims worker name) expect=reject AMQPConnectionError Server communication error: SSLError(1, '[SSL: TLSV1_ALERT_U
|
||||
PASS D svc -> rabbitmq:5671 same CA, unknown identity intruder expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
|
||||
PASS D svc -> rabbitmq:5671 same CA, identity lb expect=reject ProbableAuthenticationError ACCESS_REFUSED - Login was refused using authentication mech
|
||||
PASS D api AMQP rights publish to exchange attachments expect=allow ok
|
||||
PASS D api AMQP rights consume attachments.process expect=deny ChannelAccessRefused ACCESS_REFUSED - read access to queue 'attachments.process' in vhost '
|
||||
PASS D api AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
|
||||
PASS D api AMQP rights publish via default exchange expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'amq.default' in vhost 'mess
|
||||
PASS D worker AMQP rights consume attachments.process expect=allow ok
|
||||
PASS D worker AMQP rights publish to exchange attachments expect=deny ChannelAccessRefused ACCESS_REFUSED - write access to exchange 'attachments' in vhost 'mess
|
||||
PASS D worker AMQP rights declare a queue (configure) expect=deny ChannelAccessRefused ACCESS_REFUSED - configure access to queue 'audit-probe' in vhost 'mes
|
||||
PASS E svc -> otel-collector:4318 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
|
||||
PASS E svc -> otel-collector:4318 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS E svc -> otel-collector:4318 root CA identity lb (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS E svc -> otel-collector:4318 root CA identity api (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS E svc -> otel-collector:4318 telemetry CA (not ingest CA) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS E svc -> otel-collector:4318 ingest CA api.otel expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS E svc -> otel-collector:4318 ingest CA worker.otel expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> prometheus:9090 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
|
||||
PASS F -> prometheus:9090 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> prometheus:9090 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> prometheus:9090 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> prometheus:9090 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> prometheus:9090 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> prometheus:9090 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> loki:3100 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
|
||||
PASS F -> loki:3100 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> loki:3100 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> loki:3100 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> loki:3100 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> loki:3100 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> loki:3100 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> tempo:3200 no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
|
||||
PASS F -> tempo:3200 foreign CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> tempo:3200 root CA identity api expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> tempo:3200 ingest CA (apps cannot bypass collector) expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F -> tempo:3200 telemetry CA collector.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> tempo:3200 telemetry CA grafana.out expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F -> tempo:3200 telemetry CA operator expect=accept accepted: HTTP/1.1 200 OK
|
||||
PASS F collector -> tempo:4317 (gRPC) no client cert expect=reject rejected: after handshake: TLSV13_ALERT_CERTIFICATE_REQUIRED
|
||||
PASS F collector -> tempo:4317 (gRPC) ingest CA expect=reject rejected: after handshake: TLSV1_ALERT_UNKNOWN_CA
|
||||
PASS F collector -> tempo:4317 (gRPC) telemetry CA collector.out expect=accept accepted: | ||||