Compare commits

2 Commits
Author SHA1 Message Date
romashko 442595b37c Grade 4 2026-11-16 16:08:15 +03:00
romashko 0497b0c217 Add 5 lab code 2026-11-16 15:13:01 +03:00
63 changed files with 5100 additions and 53 deletions
+19 -21
View File
@@ -1,30 +1,28 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses.
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
MAX_PENDING_ATTACHMENTS=64
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_MEMORY_MB=768
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Choose credentials locally.
POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace-with-local-password
CACHE_TTL_SECONDS=30
MAX_REQUEST_BYTES=65536
MAX_INFLIGHT=32
MINIO_ROOT_USER=local-admin
MINIO_ROOT_PASSWORD=replace-with-admin-password
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=replace-with-api-password
# Compose sets service.name separately for API, dispatcher and worker.
OTEL_SDK_DISABLED=false
GRAFANA_ADMIN_PASSWORD=replace-with-grafana-password
+2
View File
@@ -0,0 +1,2 @@
# Shell scripts are mounted into Linux containers, including on Windows hosts.
*.sh text eol=lf
+28 -14
View File
@@ -1,18 +1,32 @@
## Лабораторная 6
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Ромашко Владимир, БВТ2301.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Целевая оценка (3/4/5): **4**.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Что реализовано и что перенесено: приложение lab5 с API-репликами, БД, S3, брокером и worker сохранено. Добавлены OTLP metrics/logs/traces, Collector, Prometheus/Loki/Tempo, Grafana provisioning, сквозной trace context с retry, SLO и два алерта.
Команды запуска из чистого clone:
```sh
cp .env.example .env
# Задать локальные пароли в .env.
docker compose up -d --build
curl http://127.0.0.1:8080/health/ready # дождаться 200
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
python -X utf8 student-tests/observability.py --scenario all
```
Результаты проверок: check, публичный smoke и проверка сигналов — PASS; сценарии worker/БД/S3/Tempo пройдены, алерты firing → resolved. Нагрузка: 2389 запросов за 30s, без ошибок. [REPORT.md](../REPORT.md), [вывод](../evidence/verification.txt), [сценарии](../evidence/observability.json), [нагрузка](../evidence/load.json).
Ограничения / известные ошибки: один узел, без HA/TLS и внешней доставки алертов; при длительном отказе backend допустима потеря телеметрии. Блокирующих ошибок в проверенных сценариях не выявлено.
Источники и использование ИИ: контракт лабораторной, документация OpenTelemetry и Loki; Codex — реализация, тестирование и оформление. Ссылки в REPORT.md.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены эквивалентными Python-командами; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+24
View File
@@ -0,0 +1,24 @@
FROM python:3.12.8-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY src ./src
COPY migrations ./migrations
RUN useradd --create-home --shell /usr/sbin/nologin appuser
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+49 -14
View File
@@ -1,37 +1,72 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Ромашко Владимир, БВТ2301.
- Целевая оценка: **4**.
- Источники: [контракт](contracts/observability.md), [OpenTelemetry](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/). Библиотеки: FastAPI, SQLAlchemy, Redis, boto3, Pillow, OpenTelemetry SDK, psutil. Codex использовался для реализации, проверок и оформления.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Compose и Python 3.9+ на хосте; свободные порты 8080, 3000, 9090, 3100, 3200. В контейнерах Python 3.12.8; PostgreSQL 17.4, Redis 7.4.2, Nginx 1.28.0; MinIO закреплён digest. Телеметрия: SDK 1.38.0 / semantic conventions 0.59b0, Collector 0.136.0, Prometheus 3.6.0, Loki 3.5.7, Tempo 2.9.0, Grafana 12.2.0.
```sh
cp .env.example .env
# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD,
# S3_SECRET_ACCESS_KEY и GRAFANA_ADMIN_PASSWORD.
docker compose config --quiet
docker compose up -d --build
curl http://127.0.0.1:8080/health/ready # дождаться 200 {"status":"ok"}
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
python -X utf8 student-tests/observability.py --scenario all
python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
docker compose down # безопасная остановка, volumes остаются
```
Миграции (включая 008 с trace context) и создание S3 bucket выполняются автоматически. Grafana: http://localhost:3000, `admin`, пароль из `.env`; dashboard **Lab 6 / Messenger / Lab 6** импортируется автоматически. Python-команды check/smoke эквивалентны `make check` / `make test`; на проверявшемся Windows-хосте make отсутствует. Для **намеренного удаления всех учебных данных**: `docker compose down -v` (при проверках не выполнялось).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Из lab5 сохранены Nginx → две API-реплики, PostgreSQL, Redis для сессий/кеша, S3, durable Redis Streams, outbox/dispatcher и worker. PostgreSQL — источник истины для заданий; attachment и outbox фиксируются одной транзакцией после записи staging, ack выполняется после устойчивого результата.
Добавлены API/worker/dispatcher → OTLP → Collector → Prometheus (метрики), Loki (логи), Tempo (трассы) → Grafana. История хранится в volumes, retention 72 часа. W3C context сохраняется в outbox и переносится через broker в worker и image process; retry остаётся в той же трассе. Экспортируются шаблоны маршрутов и технические поля, без SQL-параметров, токенов и содержимого сообщений.
Метрики: `http_requests_total` и `jobs_processed_total` — счётчики; `http_duration_seconds`, `job_duration_seconds`, `attachment_time_to_ready_seconds` — histogram в секундах; `queue_depth`, `db_pool_used/waiting`, `worker_busy/capacity` — количества; `queue_oldest_age_seconds` — секунды; `container_memory_bytes` / `container_cpu_seconds_total` — память/CPU. Точные имена, labels и запросы: [описание сигналов](infra/otel/README.md#сигналы), [dashboard JSON](infra/grafana/dashboards/messenger.json).
Переменные `.env.example` подставляются в Compose: пароли — соответствующим сервисам, `MAX_*`, `JOB_*`, `WORKER_CONCURRENCY` — приложению, `APP_PORT` — Nginx. `OTEL_SDK_DISABLED=false` включает телеметрию; service.name и внутренние адреса заданы отдельно по ролям.
SLI = 1 − 5xx / все запросы без health за **30 минут**, SLO ≥ **99%**. `ApiHighErrorRate`: >5% ошибок за 2m, минимум 20 запросов, `for: 20s`. `QueueLagHigh`: возраст queued >30s при свежем sample, `for: 15s`. Правила и [инструкции диагностики](infra/otel/README.md#queue-lag) сохранены в репозитории.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура и публичный smoke | Команды check/smoke выше | PASS при включённой телеметрии | [verification.txt](evidence/verification.txt) |
| Минимум lab5 и общий стенд | `docker compose up -d --build`; smoke | API, upload 202, ready, оригинал и thumbnail работают; хранилища имеют volumes | [compose.yaml](compose.yaml), verification.txt |
| OTLP: метрики, логи, трассы API/worker | `observability.py --scenario smoke` | Все обязательные метрики найдены; связанные логи трёх сервисов | [observability.json: smoke](evidence/observability.json) |
| Импортируемый dashboard и безопасные атрибуты | Provisioning; сценарий smoke | 18 панелей, 3 datasource; проверены labels и отсутствие приватных входных данных | [provisioning](infra/grafana/provisioning), observability.json |
| Корреляция HTTP/job → log/trace | Сценарий smoke | Найдены request_id, trace_id и parent-связи до S3/DB | observability.json: smoke |
| W3C propagation, включая retry | `observability.py --scenario retry` | Попытки 1 и 2 в одной трассе, tracestate сохранён | observability.json: retry |
| p50/p95/p99, очередь, DB pool, worker, CPU/RAM, time-to-ready | Сценарий smoke; dashboard | Метрики существуют; percentile вычисляется по сумме histogram buckets | dashboard JSON, observability.json |
| SLI/SLO и два проверенных алерта | `--scenario queue`, `--scenario database` | Оба алерта firing → resolved после восстановления | [alerts.yaml](infra/otel/alerts.yaml), observability.json |
| Ограниченная буферизация и отказ telemetry backend | `--scenario backend` | API продолжает работу; очередь ≤32 batches, RSS ≤199.03 MiB | [collector.yaml](infra/otel/collector.yaml), observability.json |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Здесь `observability.py` — `python -X utf8 student-tests/observability.py`. Сценарии останавливают реальные контейнеры текущего Compose-проекта и восстанавливают их в `finally`; volumes не удаляют.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Проверки 16.11.2026. Гипотеза: телеметрия показывает причину отказа, а её backend не блокирует бизнес-операции. Docker Desktop: 8 CPU, 7.76 GiB RAM; лимиты API — по 1 CPU/512 MiB, worker — 2 CPU/2 GiB, Collector — 0.5 CPU/256 MiB.
Нагрузка: один прогон closed-loop POST сообщений, 4 клиента, 30 секунд, без отдельного прогрева и автоматических retry. **2389 успешных запросов, 0 ошибок, 79.219 RPS; p50/p95/p99 = 40.784/99.520/129.314 ms**. Исходная сводка: [load.json](evidence/load.json). Это локальная проверка, не исследование предельной производительности; CPU во время прогона отдельно не фиксировался, RSS Collector измерен в сценарии отказа ниже.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановить worker | Рост возраста очереди и QueueLagHigh | Depth=1, возраст 55s; producer есть, consumer появляется после запуска; затем ready, depth=0 | firing → resolved: 20.3s; принятое вложение сохранено |
| Остановить PostgreSQL | 503 и ApiHighErrorRate | Error ratio 42.6%; в trace `db.acquire`/`OperationalError` и связанный лог — отказ БД, а не медленный handler | После запуска запросы успешны; firing → resolved: 109.1s с учётом окна 2m |
| Временно остановить S3 | Retry без потери задания | Первая попытка failed, вторая ready, общий trace | Задание успешно завершено |
| Остановить Tempo на 60s | Приложение работает, буферы ограничены | 165 записей и обработка вложения успешны; очередь ≤32, RSS 198.52–199.02 MiB; после запуска новая трасса найдена | Потеря 808 принятых Collector spans, бизнес-операции успешны |
Порядок диагностики: dashboard → лог по request_id/trace_id → ошибочный или отсутствующий span зависимости → восстановление сервиса → проверка ready/успешных запросов и resolved. Для Tempo потеря рассчитана как Δaccepted − Δsent после опустошения очереди; повторы отправки не считаются отдельными потерянными spans. Подробные timestamps и ID — в [observability.json](evidence/observability.json).
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Один локальный узел, без HA и TLS; алерты только локальные. SDK ограничивает очереди 512 записями, Collector использует memory limiter/batch и retry до 15s: длительный отказ может терять телеметрию. Потери до приёма Collector указанной дельтой не измеряются. История удаляется по retention; метрики не заменяют журнал транзакций. При недоступной БД API возвращает 503, после восстановления работает снова. Треки оценки 5 не реализовывались.
+313
View File
@@ -0,0 +1,313 @@
x-api: &api
image: mtusi2026lab6-api:local
pull_policy: never
dns_opt:
- timeout:1
- attempts:1
environment: &app-env
APP_HOST: 0.0.0.0
APP_PORT: 8080
POSTGRES_HOST: postgres
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_PENDING_ATTACHMENTS: ${MAX_PENDING_ATTACHMENTS:-64}
BROKER_URL: redis://broker:6379/0
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
OTEL_SERVICE_NAME: messenger-api
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
networks: [backend]
cpus: 1.0
mem_limit: 512m
stop_grace_period: 10s
services:
broker:
image: redis:7.4.2-bookworm
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction]
volumes:
- broker_data:/data
networks: [backend]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 256m
dispatcher:
<<: *api
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-dispatcher
command: [python, -m, src.jobs, dispatch]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
mem_limit: 128m
worker:
<<: *api
environment:
<<: *app-env
OTEL_SERVICE_NAME: messenger-worker
command: [python, -m, src.worker]
init: true
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 2.0
mem_limit: 2g
pids_limit: 64
api:
<<: *api
build: .
deploy:
replicas: 2
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-init:
condition: service_completed_successfully
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
interval: 2s
timeout: 2s
start_period: 5s
retries: 2
migrate:
<<: *api
environment:
<<: *app-env
OTEL_SDK_DISABLED: 'true'
command: [python, -m, src.database]
healthcheck:
disable: true
restart: 'no'
depends_on:
postgres:
condition: service_healthy
s3:
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
command: [server, /data, --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
MINIO_BROWSER: 'off'
volumes:
- s3_data:/data
networks: [backend]
healthcheck:
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
s3-init:
# The pinned server image also contains mc; no second image is needed.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
entrypoint: [sh, /bootstrap.sh]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
volumes:
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
- ./infra/s3-policy.json:/policy.json:ro
depends_on:
s3:
condition: service_healthy
networks: [backend]
restart: 'no'
cpus: 0.25
mem_limit: 128m
cleanup:
<<: *api
environment:
<<: *app-env
OTEL_SDK_DISABLED: 'true'
command: [python, -m, src.cleanup]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
nginx:
image: nginx:1.28.0-alpine
entrypoint: [sh, /etc/nginx/readiness.sh]
dns_opt:
- timeout:1
- attempts:1
ports:
- '127.0.0.1:${APP_PORT:-8080}:8080'
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/readiness.sh:/etc/nginx/readiness.sh:ro
networks: [edge, backend]
depends_on:
api:
condition: service_healthy
healthcheck:
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 2s
timeout: 2s
retries: 3
cpus: 0.5
mem_limit: 128m
stop_grace_period: 10s
postgres:
image: postgres:17.4-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
volumes:
- postgres_data:/var/lib/postgresql/data
networks: [backend]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
redis:
image: redis:7.4.2-bookworm
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
volumes:
- redis_data:/data
networks: [backend]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 256m
otel-collector:
image: otel/opentelemetry-collector-contrib:0.136.0
command: [--config=/etc/otel/collector.yaml]
volumes:
- ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro
networks: [backend]
mem_limit: 256m
cpus: 0.5
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.6.0
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=3d]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
ports: ['127.0.0.1:9090:9090']
networks: [edge, backend]
mem_limit: 512m
cpus: 0.5
restart: unless-stopped
loki:
image: grafana/loki:3.5.7
command: [-config.file=/etc/loki/local.yaml]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro
- loki_data:/loki
ports: ['127.0.0.1:3100:3100']
networks: [edge, backend]
mem_limit: 384m
cpus: 0.5
restart: unless-stopped
tempo:
image: grafana/tempo:2.9.0
command: [-config.file=/etc/tempo.yaml]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro
- tempo_data:/var/tempo
ports: ['127.0.0.1:3200:3200']
networks: [edge, backend]
mem_limit: 384m
cpus: 0.5
restart: unless-stopped
grafana:
image: grafana/grafana:12.2.0
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
GF_USERS_ALLOW_SIGN_UP: 'false'
GF_ANALYTICS_REPORTING_ENABLED: 'false'
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
ports: ['127.0.0.1:3000:3000']
networks: [edge, backend]
mem_limit: 384m
cpus: 0.5
restart: unless-stopped
networks:
edge:
backend:
internal: true
volumes:
broker_data:
postgres_data:
redis_data:
s3_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.157,
"concurrency": 4,
"successful_requests": 2389,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 79.219,
"successful_latency_ms": {
"p50": 40.784,
"p95": 99.52,
"p99": 129.314
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
File diff suppressed because it is too large Load Diff
+5
View File
@@ -0,0 +1,5 @@
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
Running smoke
PASS: smoke
PASS: Grafana provisioning (18 panels, 3 datasources), all dashboard queries, no active alerts; local secrets absent from Git candidates; assignment/contracts/public tests unchanged.
+716
View File
@@ -0,0 +1,716 @@
{
"uid": "messenger-lab6",
"title": "Messenger / Lab 6",
"schemaVersion": 41,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-30m",
"to": "now"
},
"tags": [
"lab6"
],
"panels": [
{
"id": 1,
"title": "API requests / second",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"description": "Business requests only; health probes are excluded.",
"targets": [
{
"refId": "A",
"expr": "sum by (method,route) (rate(http_requests_total{service_name=\"messenger-api\"}[2m]))",
"legendFormat": "{{method}} {{route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 2,
"title": "API server error ratio",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"description": "5xx / all non-health requests, rolling 2m. 4xx are not server errors.",
"targets": [
{
"refId": "A",
"expr": "messenger:http_error_ratio:2m",
"legendFormat": "5xx / all"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"title": "API duration p50 / p95 / p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"description": "Quantiles from merged histogram buckets across API replicas. Never averages of per-instance percentiles.",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"title": "Job attempts / second",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"description": "One count per claimed attempt; failed includes retryable failures, skipped is a fenced attempt.",
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(jobs_processed_total[2m]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"title": "Worker processing duration p50 / p95 / p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"description": "Includes isolated image process and durable result check; excludes queue wait.",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"title": "Time from accepted upload to ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"description": "Database timestamps: durable attachment creation to ready update. Includes queue wait and retries; observed on successful attempts.",
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"title": "Durable queue depth",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"description": "PostgreSQL queued attachments, including unpublished outbox/retries; excludes processing jobs.",
"targets": [
{
"refId": "A",
"expr": "max(queue_depth)",
"legendFormat": "queued"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"title": "Oldest queued attachment age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"description": "Age since acceptance. A stale sample is not zero; check queue sample age below.",
"targets": [
{
"refId": "A",
"expr": "max(queue_oldest_age_seconds)",
"legendFormat": "age"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 9,
"title": "DB pool used / acquiring",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"description": "Per long-lived process. Acquiring includes connection creation and pool wait; max pool size is 10.",
"targets": [
{
"refId": "A",
"expr": "db_pool_used",
"legendFormat": "{{service_name}} {{service_instance_id}} used"
},
{
"refId": "B",
"expr": "db_pool_waiting",
"legendFormat": "{{service_name}} {{service_instance_id}} acquiring"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"title": "Worker saturation",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"description": "Busy consumers / configured concurrency. Missing capacity after 30s indicates stopped worker.",
"targets": [
{
"refId": "A",
"expr": "sum(worker_busy)",
"legendFormat": "busy"
},
{
"refId": "B",
"expr": "sum(worker_capacity)",
"legendFormat": "capacity"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"title": "Container memory (includes image children)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"description": "cgroup v2 memory.current. Includes child processes and charged cache.",
"targets": [
{
"refId": "A",
"expr": "container_memory_bytes",
"legendFormat": "{{service_name}} {{service_instance_id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"title": "Container CPU",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"description": "cgroup v2 CPU seconds / second; 1.0 is one fully used CPU core.",
"targets": [
{
"refId": "A",
"expr": "rate(container_cpu_seconds_total[2m])",
"legendFormat": "{{service_name}} {{service_instance_id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"title": "API availability SLI / SLO",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"description": "Target: >=99% non-5xx business requests in a rolling 30m window. No traffic is not evidence that SLO is met.",
"targets": [
{
"refId": "A",
"expr": "1 - ((sum(increase(http_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[30m])) or vector(0)) / clamp_min(sum(increase(http_requests_total{service_name=\"messenger-api\"}[30m])), 1))",
"legendFormat": "SLI over 30m"
},
{
"refId": "B",
"expr": "vector(0.99)",
"legendFormat": "SLO"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"title": "Local alerts",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"description": "1 means pending/firing. Series disappears after resolution. Rules and runbooks are stored in infra/otel.",
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertname=~\"ApiHighErrorRate|QueueLagHigh\"}",
"legendFormat": "{{alertname}} {{alertstate}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"title": "Collector memory",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"description": "",
"targets": [
{
"refId": "A",
"expr": "otelcol_process_memory_rss_bytes",
"legendFormat": "RSS"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 16,
"title": "Collector exporter queue / capacity",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"description": "Bounded queues: 32 batches per backend, up to 256 records per batch.",
"targets": [
{
"refId": "A",
"expr": "otelcol_exporter_queue_size",
"legendFormat": "{{exporter}} queued"
},
{
"refId": "B",
"expr": "otelcol_exporter_queue_capacity",
"legendFormat": "{{exporter}} capacity"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"title": "Collector failed sends / rejected enqueue",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"description": "Send failures may include retries, not unique lost spans. Exact experiment loss is measured by accepted/sent deltas after draining.",
"targets": [
{
"refId": "A",
"expr": "rate(otelcol_exporter_send_failed_spans_total[2m])",
"legendFormat": "{{exporter}} span send failures"
},
{
"refId": "B",
"expr": "rate(otelcol_exporter_send_failed_log_records_total[2m])",
"legendFormat": "{{exporter}} log send failures"
},
{
"refId": "C",
"expr": "rate(otelcol_exporter_enqueue_failed_spans_total[2m])",
"legendFormat": "{{exporter}} span queue rejections"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"title": "Queue sample age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 64,
"w": 12,
"h": 8
},
"description": "Above 15s: queue measurements are stale (e.g. DB outage).",
"targets": [
{
"refId": "A",
"expr": "time() - max(queue_sample_timestamp_seconds)",
"legendFormat": "seconds since DB sample"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
}
],
"links": [
{
"title": "Explore logs",
"url": "/explore?left=%7B%22datasource%22%3A%22loki%22%7D"
},
{
"title": "Explore traces",
"url": "/explore?left=%7B%22datasource%22%3A%22tempo%22%7D"
}
]
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: messenger
folder: Lab 6
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,41 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: trace_id
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: '$${__value.raw}'
urlDisplayLabel: View trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-5m'
spanEndTimeShift: '5m'
tags: [{key: service.name, value: service_name}]
filterByTraceID: true
filterBySpanID: false
nodeGraph:
enabled: true
+38
View File
@@ -0,0 +1,38 @@
worker_processes 1;
worker_shutdown_timeout 8s;
events { worker_connections 1024; }
http {
upstream messenger {
zone messenger 64k;
include /etc/nginx/ready-backends.conf;
}
log_format upstream '$request_method $uri $status upstream=$upstream_addr '
'upstream_status=$upstream_status time=$request_time';
access_log /var/log/nginx/access.log upstream;
server {
listen 8080;
# API enforces the byte limit and returns the contracted JSON error.
client_max_body_size 0;
client_body_timeout 5s;
client_header_timeout 5s;
keepalive_timeout 5s;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_request_buffering off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header Connection close;
proxy_connect_timeout 1s;
proxy_send_timeout 35s;
proxy_read_timeout 120s;
# A 503 is an API response (for example, S3 is unavailable), so
# preserve it for the client instead of marking every replica down.
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 8s;
# Never transparently retry a POST already sent to an API.
# A client retries messages using the original Idempotency-Key.
}
}
}
+137 -4
View File
@@ -1,7 +1,140 @@
# Конфигурация наблюдаемости — реализовать
# Лаба 6: запуск и диагностика
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
```sh
cp .env.example .env
# Задать локальные пароли в .env (в Git не добавлять).
docker compose up -d --build
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
python -X utf8 student-tests/observability.py --scenario all
```
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
Последняя команда создаёт синтетические данные и временно останавливает **только
сервисы текущего Compose-проекта**. В `finally` запускает остановленные сервисы.
Не запускать параллельно с другими проверками этого стенда. Volumes не удаляются.
Санитизированные результаты сохраняются в `evidence/observability.json`.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
Grafana: http://localhost:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD`
из `.env`. Dashboard **Lab 6 / Messenger / Lab 6** и три datasource загружаются
автоматически. Prometheus: http://localhost:9090; Loki API: порт 3100; Tempo API:
3200. Все опубликованные порты слушают только loopback, OTLP наружу не опубликован.
Collector принимает OTLP/HTTP от API, dispatcher и worker. Prometheus хранит
метрики, Loki — логи, Tempo — трассы; Grafana обращается к этим хранилищам.
История сохраняется в именованных volumes, retention — 72 часа.
Версии закреплены в Compose и requirements.txt: OTel Python 1.38.0, semantic
conventions 0.59b0 (зависимость SDK); наши имена метрик перечислены ниже.
Это явная инструментация: auto-instrumentation, сбор SQL и HTTP headers отключены.
## Сигналы
| Prometheus name | Единицы и смысл |
| --- | --- |
| `http_requests_total` | Число бизнес-запросов; method, route template, status_class |
| `http_duration_seconds` | Histogram полного HTTP-запроса, включая отдачу stream |
| `jobs_processed_total` | Попытки обработки; job_type, outcome=ready/failed/skipped |
| `job_duration_seconds` | Histogram времени попытки, без ожидания в очереди |
| `attachment_time_to_ready_seconds` | Histogram от записи attachment до durable ready, включая retry |
| `queue_depth` | Количество queued в PostgreSQL, включая outbox и отложенные retry |
| `queue_oldest_age_seconds` | Возраст старейшего queued от момента приёма, не время обработки |
| `queue_sample_timestamp_seconds` | Unix-время последнего успешного опроса очереди |
| `db_pool_used`, `db_pool_waiting` | Занятые соединения / потоки, получающие соединение (ожидание + reconnect) |
| `worker_busy`, `worker_capacity` | Число занятых consumer / лимит concurrency |
| `process_memory_bytes`, `process_cpu_seconds_total` | RSS и CPU процесса |
| `container_memory_bytes`, `container_cpu_seconds_total` | cgroup v2, включая дочерние процессы обработки |
Histogram buckets, секунды: .005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10,
30, 60, 120, 300, +Inf. Dashboard содержит PromQL, единицы и описания каждого
графика. p50/p95/p99 считаются `histogram_quantile` по сумме buckets всех реплик.
Метрики очереди отдаёт только dispatcher; это не длина Redis Stream (в нём
ещё могут быть уже взятые сообщения). При недоступной БД sample становится
устаревшим; возраст sample выведен отдельно, QueueLagHigh не использует stale data.
Resource: service.name, service.version=1.6.0, service.instance.id=hostname,
deployment.environment.name=lab. API: messenger-api; worker и его дочерний
процесс: messenger-worker; publisher: messenger-dispatcher. Дочерний процесс
экспортирует только traces/logs, чтобы не создавать отдельные краткоживущие
ряды метрик. Parent измеряет длительность попытки и результат после COMMIT.
Метрики наблюдательные: авария после COMMIT до записи счётчика может потерять
измерение, но не результат обработки. Предметных ID в labels метрик нет.
## Связь сигналов
SERVER span API → traceparent/tracestate в transactional outbox → PRODUCER span
dispatcher → W3C context в JSON сообщения Redis → CONSUMER span worker → context
через окружение изолированного image process → S3/DB CLIENT spans. Retry публикует
новый producer/consumer в той же трассе, с `job.attempt`. Baggage не переносится.
Context сохранён в миграции 008, поэтому рестарт API не разрывает причинную связь.
Логи содержат timestamp/severity/service.name/event и trace_id/span_id активного
span. `http_request` содержит ограниченный `request_id`, совпадающий с
X-Request-Id ответа. Loki сохраняет технические поля как structured metadata;
индексирует только service.name и окружение. Для корреляции достаточно request_id,
trace_id и номера попытки: имена файлов и предметные ID не экспортируются.
В Grafana Explore / Loki:
```logql
{service_name="messenger-api"} | request_id="ЗНАЧЕНИЕ_X_REQUEST_ID"
{service_name=~"messenger-.*"} | trace_id="TRACE_ID"
```
Поле trace_id открывает Tempo; из Tempo доступен переход к логам. В трассе upload
видны `POST /api/v1/chats/{chat_id}/attachments`, `attachment-jobs publish`,
`attachment-jobs process`, `image.process`, `s3.GetObject`, `s3.PutObject`,
`db.SELECT`/`db.UPDATE`. Исключения содержат только тип, без message/stack/SQL.
## SLI/SLO и alerts
SLI доступности = 1 − число 5xx / число всех бизнес-запросов за скользящие **30 минут**.
SLO: **не менее 99%**. Health исключён из числителя и знаменателя; 4xx не считается
отказом сервера. Без запросов SLI не оцениваем. Это учебное окно, а не месячный SLO.
`ApiHighErrorRate`: 5xx > 5% за 2 минуты, минимум 20 запросов, `for: 20s`.
`QueueLagHigh`: queued > 0 и возраст > 30s, свежий sample, `for: 15s`.
Пороги короткие для воспроизводимой локальной демонстрации; алерты не являются
burn-rate исследованием. Rules вычисляет Prometheus; firing/pending видны в
http://localhost:9090/alerts и на dashboard. После восстановления alert пропадает
из instant `ALERTS`; история переходов остаётся в range query. Доставка наружу не настроена.
## Queue lag
1. Открыть queue depth/age и worker capacity/busy. Растущая очередь при отсутствии
capacity означает остановку worker; свежий sample подтверждает доступность БД.
2. В Loki найти `job_published`, затем открыть trace: producer есть, consumer ещё нет.
3. `docker compose start worker`; дождаться `job_processed` outcome=ready, depth=0.
4. Проверить, что QueueLagHigh больше не firing и в той же трассе появился consumer.
## API errors
1. При ApiHighErrorRate найти http_request со status_class=5xx и взять trace_id.
2. В Tempo проверить `db.acquire`/`db.SELECT`: error.type и длительность покажут
отказ подключения/БД. В Loki есть `db_error`/`operation_failed`. Медленный handler
без отказа зависимости не имел бы ошибочного DB span. Посмотреть db_pool_waiting.
3. Проверить PostgreSQL: `docker compose ps postgres`; восстановить
`docker compose start postgres`, дождаться readiness и успешного бизнес-запроса.
4. Через окно 2m error ratio снизится, alert перейдёт в resolved. Sample очереди
тоже должен стать свежим. Сценарий проверки сохраняет трассу ошибочного запроса.
## Недоступность telemetry backend
SDK: очереди по 512 spans/logs, batch 64, HTTP timeout 1s; metric export каждые 5s.
Отправка выполняется в фоновых потоках. Collector: memory limiter 192 MiB,
spike 48 MiB, container limit 256 MiB; batch ≤256 записей; отдельная очередь
32 batches / 2 consumers на Loki и Tempo; retry до 15s, timeout 2s, overflow
не блокирует приём. Конечные ограничения защищают память, но допускают потерю
телеметрии при длительном отказе. API и pipeline не зависят от готовности backend.
Сценарий `backend` останавливает Tempo, продолжает создавать сообщения и
обрабатывать изображения, снимает RSS/очередь Collector, затем запускает Tempo
и проверяет новую трассу. `send_failed_spans` может считать повторные попытки:
его нельзя выдавать за число уникально потерянных spans. Для отдельного окна
тест вычисляет accepted − sent после опустошения очереди (без рестартов Collector)
и сохраняет эти дельты вместе с наблюдаемой памятью. Это потери уже принятых
Collector spans; отброшенные до приёма SDK/receiver в этот счёт не входят.
Prometheus всё это время
работает, поэтому сохраняется история отказа и восстановления.
Первичные источники: [Collector](https://opentelemetry.io/docs/collector/configuration/),
[native Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
[bounded exporter queues](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.136.0/exporter/exporterhelper/README.md).
+28
View File
@@ -0,0 +1,28 @@
groups:
- name: messenger
rules:
- record: messenger:http_error_ratio:2m
expr: |
(sum(rate(http_requests_total{service_name="messenger-api",status_class="5xx"}[2m])) or vector(0))
/ clamp_min(sum(rate(http_requests_total{service_name="messenger-api"}[2m])), 0.001)
- alert: ApiHighErrorRate
expr: messenger:http_error_ratio:2m > 0.05 and sum(increase(http_requests_total{service_name="messenger-api"}[2m])) >= 20
for: 20s
labels:
severity: warning
annotations:
summary: 'API: more than 5% server errors over 2 minutes'
description: 'SLI is 5xx / all non-health requests. At least 20 requests in the window.'
runbook: 'infra/otel/README.md#api-errors'
- alert: QueueLagHigh
expr: |
max(queue_oldest_age_seconds) > 30
and max(queue_depth) > 0
and time() - max(queue_sample_timestamp_seconds) < 15
for: 15s
labels:
severity: warning
annotations:
summary: 'Oldest queued attachment waits more than 30 seconds'
description: 'Check worker capacity/busy and job errors; backlog is sampled by dispatcher.'
runbook: 'infra/otel/README.md#queue-lag'
+69
View File
@@ -0,0 +1,69 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 48
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
enable_open_metrics: true
metric_expiration: 30s
translation_strategy: UnderscoreEscapingWithSuffixes
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 2
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
queue_size: 32
num_consumers: 2
block_on_overflow: false
retry_on_failure:
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
+37
View File
@@ -0,0 +1,37 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 72h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+12
View File
@@ -0,0 +1,12 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: messenger
static_configs:
- targets: ['otel-collector:8889']
- job_name: collector
static_configs:
- targets: ['otel-collector:8888']
+23
View File
@@ -0,0 +1,23 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
trace_idle_period: 10s
max_block_duration: 5m
compactor:
compaction:
block_retention: 72h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
# Only ready API addresses enter the upstream, including at nginx startup.
update_backends() {
next=/etc/nginx/ready-backends.conf.next
: > "$next"
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
echo "server $address:8080;" >> "$next"
fi
done
# If every running API temporarily loses a shared dependency (for example
# S3), retain the last addresses. The API can then return the contracted
# 503 instead of Nginx replacing it with a 502. At initial startup there
# is no previous healthy set, so use an inert placeholder.
if [ ! -s "$next" ]; then
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
rm -f "$next"
return
fi
echo 'server 127.0.0.1:9 down;' > "$next"
fi
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
mv "$next" /etc/nginx/ready-backends.conf
if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then
nginx -s reload
fi
else
rm -f "$next"
fi
}
update_backends
(
while sleep 2; do
update_backends reload
done
) &
exec nginx -g 'daemon off;'
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
mc mb --ignore-existing local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
+17
View File
@@ -0,0 +1,17 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/variants/*"]
}
]
}
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
CREATE INDEX chats_order_idx ON chats (created_at, id);
+1
View File
@@ -0,0 +1 @@
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
+9
View File
@@ -0,0 +1,9 @@
-- Additive migration: no rewrite of existing lab2 tables or messages.
CREATE TABLE message_idempotency (
actor_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (actor_id, chat_id, key)
);
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
object_key text NOT NULL UNIQUE,
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
+23
View File
@@ -0,0 +1,23 @@
ALTER TABLE attachments
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
ADD COLUMN status text NOT NULL DEFAULT 'ready'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD COLUMN staging_key text,
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
ADD COLUMN error_code text;
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
attempt bigint NOT NULL DEFAULT 0,
failures bigint NOT NULL DEFAULT 0,
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
lease_until timestamptz,
lease_token uuid,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
replay_audit jsonb NOT NULL DEFAULT '[]'
);
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at);
+1
View File
@@ -0,0 +1 @@
ALTER TABLE attachment_jobs ADD COLUMN last_error text;
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz;
CREATE INDEX attachments_pending_idx ON attachments(status)
WHERE status IN ('queued', 'processing');
+1
View File
@@ -0,0 +1 @@
ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}';
+23
View File
@@ -0,0 +1,23 @@
fastapi==0.115.6
uvicorn==0.34.0
pydantic==2.13.5
starlette==0.41.3
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
colorama==0.4.6; sys_platform == "win32"
h11==0.16.0
idna==3.20
pydantic_core==2.46.5
typing-inspection==0.4.4
typing_extensions==4.16.0
psycopg[binary]==3.2.6
SQLAlchemy==2.0.41
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.70
Pillow==12.0.0
opentelemetry-api==1.38.0
opentelemetry-sdk==1.38.0
opentelemetry-exporter-otlp-proto-http==1.38.0
psutil==7.1.2
View File
+14
View File
@@ -0,0 +1,14 @@
import os
import uvicorn
if __name__ == '__main__':
uvicorn.run(
'src.main:app',
host=os.getenv('APP_HOST', '0.0.0.0'),
port=int(os.getenv('APP_PORT', '8080')),
workers=1,
access_log=False, # Structured logs use route templates, never query strings.
timeout_graceful_shutdown=5,
)
+21
View File
@@ -0,0 +1,21 @@
"""A shared admission limit for unfinished attachments across all API replicas."""
import os
from sqlalchemy import func, select
from .errors import APIError
from .models import Attachment
MAX_PENDING = int(os.getenv('MAX_PENDING_ATTACHMENTS', '64'))
if MAX_PENDING < 1:
raise ValueError('MAX_PENDING_ATTACHMENTS must be positive')
def reserve_capacity(session):
# Keep the lock until the new queued row commits. Counting without this lock
# would allow simultaneous uploads on different replicas to exceed the limit.
session.execute(select(func.pg_advisory_xact_lock(202605)))
count = session.scalar(select(func.count()).select_from(Attachment).where(
Attachment.status.in_(['queued', 'processing'])))
if count >= MAX_PENDING:
raise APIError(429, 'rate_limited', 'Очередь обработки заполнена; повторите загрузку позже')
+88
View File
@@ -0,0 +1,88 @@
import os
import unicodedata
from uuid import uuid4
from PIL import Image, UnidentifiedImageError
from sqlalchemy import func, select
from .errors import APIError
from .admission import reserve_capacity
from .models import Attachment, AttachmentJob, ChatMember
from .object_storage import BUCKET, PREFIX, s3_client
from .store import ensure_uuid, serialize, store
from .telemetry import carrier
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
if MAX_BYTES < 1 or MAX_PIXELS < 1:
raise ValueError('Upload limits must be positive')
def validate_filename(filename):
if (not filename or not filename.strip() or len(filename) > 255
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
raise APIError(400, 'invalid_request', 'Неверное имя файла')
def validate_image(source, content_type):
try:
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
if picture.width * picture.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
if Image.MIME[picture.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
picture.verify()
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
except Image.DecompressionBombError as exc:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
source.seek(0)
def lock_object(session, key):
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
def metadata(attachment):
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
'content_type', 'size_bytes', 'sha256', 'created_at',
'updated_at', 'status', 'error_code')
return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'}
for item in attachment.variants]}
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
validate_image(source, content_type)
attachment_id = str(uuid4())
key = PREFIX + attachment_id
staging_key = 'staging/' + attachment_id
with store.sessions.begin() as session:
store._chat(session, chat_id, actor_id)
reserve_capacity(session)
lock_object(session, staging_key)
s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source,
ContentLength=size, ContentType=content_type)
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
object_key=key, filename=filename, content_type=content_type,
size_bytes=size, sha256=digest, staging_key=staging_key)
session.add(attachment)
session.flush()
session.add(AttachmentJob(attachment_id=attachment_id, trace_context=carrier()))
result = metadata(attachment)
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
return result
def get_attachment(attachment_id, actor_id):
with store.sessions() as session:
attachment = session.scalar(select(Attachment).join(
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
if attachment is None:
raise APIError(404, 'not_found', 'Объект не найден')
return attachment
+68
View File
@@ -0,0 +1,68 @@
import hashlib
import json
import os
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from redis import Redis
from .errors import APIError
password_hasher = PasswordHasher()
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
def verify_password(encoded, password):
try:
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
except VerificationError:
return False
def unauthorized():
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
class Sessions:
def __init__(self):
self.client = Redis.from_url(
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
retry_on_timeout=False,
)
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
if self.ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
@staticmethod
def key(token):
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise unauthorized()
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
def create(self, user):
token = secrets.token_urlsafe(32)
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
pxat=int(expires.timestamp() * 1000))
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
def user_id(self, token):
raw = self.client.get(self.key(token))
if raw is None:
raise unauthorized()
session = json.loads(raw)
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
raise unauthorized()
return session['user_id']
def delete(self, token):
if not self.client.delete(self.key(token)):
raise unauthorized()
sessions = Sessions()
+40
View File
@@ -0,0 +1,40 @@
"""History pages are disposable; PostgreSQL owns both data and their version."""
import hashlib
import json
import os
from redis.exceptions import RedisError
from .auth import sessions
class HistoryCache:
def __init__(self, client, ttl=30, prefix='history'):
self.client = client
self.ttl = ttl
self.prefix = prefix
if ttl < 1:
raise ValueError('CACHE_TTL_SECONDS must be positive')
def key(self, chat_id, version, limit, after):
page = json.dumps([limit, after], separators=(',', ':'))
digest = hashlib.sha256(page.encode()).hexdigest()
return f'{self.prefix}:{chat_id}:{version}:{digest}'
def get_or_load(self, chat_id, version, limit, after, load):
key = self.key(chat_id, version, limit, after)
try:
cached = self.client.get(key)
except RedisError:
return load()
if cached is not None:
return json.loads(cached)
page = load()
try:
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
except RedisError:
pass
return page
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
+86
View File
@@ -0,0 +1,86 @@
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import os
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, s3_client
GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))))
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET):
for obj in page.get('Contents', []):
key = obj['Key']
if not key.startswith(('staging/', 'originals/', 'variants/')):
continue
with sessions.begin() as session:
lock_object(session, key)
if key.startswith('staging/'):
attachment = session.scalar(select(Attachment).where(
Attachment.staging_key == key).with_for_update())
if attachment:
if attachment.status in ('queued', 'processing'):
continue
if attachment.status == 'failed' and attachment.updated_at >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
attachment.staging_key = None
deleted += 1
continue
else:
# All final keys belong to a server-generated attachment UUID.
from uuid import UUID
try:
attachment_id = str(UUID(key.split('/')[1]))
except (ValueError, IndexError):
continue
attachment = session.get(Attachment, attachment_id, with_for_update=True)
if attachment:
if attachment.status != 'failed' or attachment.updated_at >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
return deleted
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
try:
while True:
try:
count = cleanup(sessionmaker(engine), s3_client())
logging.info('Cleanup completed: deleted=%d', count)
except Exception as exc:
logging.error('Cleanup deferred: %s', type(exc).__name__)
if args.once:
raise SystemExit(1) from None
if args.once:
break
time.sleep(60)
finally:
engine.dispose()
if __name__ == '__main__':
main()
+106
View File
@@ -0,0 +1,106 @@
import os
from pathlib import Path
from sqlalchemy import create_engine, event, func, insert, select
from sqlalchemy.engine import URL, make_url
from sqlalchemy.pool import QueuePool
from threading import Lock
from . import telemetry as otel
from .models import SchemaMigration
def database_url():
if os.getenv('DATABASE_URL'):
url = make_url(os.environ['DATABASE_URL'])
if url.drivername in ('postgres', 'postgresql'):
url = url.set(drivername='postgresql+psycopg')
if url.drivername != 'postgresql+psycopg':
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
return url
return URL.create(
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
database=os.getenv('POSTGRES_DB', 'messenger'),
username=os.getenv('POSTGRES_USER', 'messenger'),
password=os.environ['POSTGRES_PASSWORD'],
)
class ObservedPool(QueuePool):
"""Count threads acquiring a connection, including pool wait and reconnect time."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.waiting = 0
self.wait_lock = Lock()
def connect(self):
with self.wait_lock:
self.waiting += 1
try:
with otel.dependency_span('db.acquire', **{'db.system': 'postgresql'}):
return super().connect()
finally:
with self.wait_lock:
self.waiting -= 1
def create_db_engine():
engine = create_engine(
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
poolclass=ObservedPool, hide_parameters=True,
connect_args={'connect_timeout': 2,
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
)
# Only operation names; SQL text, bound values and connection URLs are never exported.
@event.listens_for(engine, 'before_cursor_execute')
def before(conn, cursor, statement, parameters, context, executemany):
if not otel.trace.get_current_span().get_span_context().is_valid:
return
operation = statement.lstrip().split(None, 1)[0].upper() if statement.strip() else 'OTHER'
if operation not in {'SELECT', 'INSERT', 'UPDATE', 'DELETE'}:
operation = 'OTHER'
context.otel_span = otel.tracer.start_span('db.' + operation,
kind=otel.SpanKind.CLIENT, attributes={'db.system': 'postgresql', 'db.operation.name': operation})
@event.listens_for(engine, 'after_cursor_execute')
def after(conn, cursor, statement, parameters, context, executemany):
current = getattr(context, 'otel_span', None)
if current:
current.end()
@event.listens_for(engine, 'handle_error')
def error(context):
current = getattr(context.execution_context, 'otel_span', None)
if current:
current.set_status(otel.Status(otel.StatusCode.ERROR))
current.set_attribute('error.type', type(context.original_exception).__name__)
current.end()
otel.event('db_error', severity=otel.logging.ERROR, error_type=type(context.original_exception).__name__)
if os.getenv('OTEL_CHILD_PROCESS') != 'true':
otel.meter.create_observable_gauge('db_pool_used', callbacks=[
lambda _: [otel.metrics.Observation(engine.pool.checkedout())]])
otel.meter.create_observable_gauge('db_pool_waiting', callbacks=[
lambda _: [otel.metrics.Observation(engine.pool.waiting)]])
return engine
def migrate(engine):
with engine.begin() as conn:
conn.execute(select(func.pg_advisory_xact_lock(202602)))
SchemaMigration.__table__.create(conn, checkfirst=True)
applied = set(conn.scalars(select(SchemaMigration.version)))
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
if path.name not in applied:
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
conn.execute(insert(SchemaMigration).values(version=path.name))
if __name__ == '__main__':
engine = create_db_engine()
try:
migrate(engine)
print('Migrations applied')
finally:
engine.dispose()
+31
View File
@@ -0,0 +1,31 @@
from typing import Optional
from fastapi import Depends, Header, Query
from .auth import sessions, unauthorized
from .errors import APIError
from .store import store
def page_limit(limit: str = Query(default="50")) -> int:
if not limit.isascii() or not limit.isdecimal() or len(limit) > 3:
raise APIError(400, "invalid_request", "Неверный limit")
number = int(limit)
if not 1 <= number <= 100:
raise APIError(400, "invalid_request", "Неверный limit")
return number
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
parts = (authorization or '').split()
if len(parts) != 2 or parts[0].lower() != 'bearer':
raise unauthorized()
return parts[1]
def current_user(token: str = Depends(bearer_token)) -> dict:
user = store.get_user(sessions.user_id(token))
if user is None:
raise unauthorized()
return user
+60
View File
@@ -0,0 +1,60 @@
from fastapi import Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from .middleware import INSTANCE_ID
class APIError(Exception):
def __init__(self, status_code: int, code: str, message: str):
self.status_code = status_code
self.code = code
self.message = message
def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse:
request_id = getattr(request.state, "request_id", "")
return JSONResponse(
status_code=status_code,
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
content={
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
},
)
def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
response = error_response(request, exc.status_code, exc.code, exc.message)
if exc.status_code == 429:
response.headers['Retry-After'] = '1'
if exc.status_code == 401:
response.headers['WWW-Authenticate'] = (
'Basic realm="messenger", charset="UTF-8"'
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
)
return response
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
return error_response(request, 400, "invalid_request", "Неверный запрос")
def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
if exc.status_code == 404:
return error_response(request, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request, 400, "invalid_request", "Метод не поддерживается")
else:
return error_response(request, exc.status_code, "invalid_request", "Неверный запрос")
def internal_error_handler(request: Request, exc: Exception) -> JSONResponse:
return error_response(request, 500, "internal_error", "Внутренняя ошибка")
+122
View File
@@ -0,0 +1,122 @@
"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC."""
import hashlib
import io
import os
import sys
from PIL import Image, ImageOps, UnidentifiedImageError
from sqlalchemy.orm import sessionmaker
from .attachments import MAX_BYTES, MAX_PIXELS
from .database import create_db_engine
from .jobs import now
from .models import Attachment, AttachmentJob
from .object_storage import BUCKET, s3_client
from . import telemetry as otel
def fit_size(size, bound):
width, height = size
if max(size) <= bound:
return size
return max(1, width * bound // max(size)), max(1, height * bound // max(size))
def variants(data):
with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original:
if original.width * original.height > MAX_PIXELS:
raise ValueError('pixel limit')
original.load()
oriented = ImageOps.exif_transpose(original)
rgba = oriented.convert('RGBA')
rgb = Image.new('RGB', rgba.size, 'white')
rgb.paste(rgba, mask=rgba.getchannel('A'))
rgba.close()
oriented.close()
rgb.info.clear()
try:
for name, image, fmt, mime, quality in (
('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80),
):
with image, io.BytesIO() as output:
image.save(output, format=fmt, quality=quality)
blob = output.getvalue()
yield {'name': name, 'content_type': mime, 'width': image.width,
'height': image.height, 'size_bytes': len(blob),
'sha256': hashlib.sha256(blob).hexdigest()}, blob
finally:
rgb.close()
def process(sessions, job_id, token):
with sessions() as session:
job = session.get(AttachmentJob, job_id)
if job.lease_token != token:
return
attachment = session.get(Attachment, job.attachment_id)
staging_key = attachment.staging_key
client = s3_client()
response = client.get_object(Bucket=BUCKET, Key=staging_key)
with response['Body'] as body:
data = body.read(MAX_BYTES + 1)
if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256:
raise ValueError('staging checksum mismatch')
try:
rendered = list(variants(data))
except OSError as exc:
raise ValueError('invalid image') from exc
# Job lock fences the final writes, cleanup and publication against lease recovery.
# Work before this transaction is CPU-only. Keys are stable across retries.
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job.lease_token != token or job.lease_until <= now():
return
current = session.get(Attachment, job.attachment_id, with_for_update=True)
if current.status != 'processing':
return
client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data,
ContentType=current.content_type)
saved = []
for meta, blob in rendered:
key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}'
client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type'])
saved.append({**meta, 'object_key': key})
# Test-only crash injection; never enabled by compose.yaml.
if os.getenv('JOB_FAULT') == 'after_objects':
os._exit(91)
current.variants = saved
current.status, current.error_code = 'ready', None
current.updated_at = now()
job.lease_token = job.lease_until = None
if os.getenv('JOB_FAULT') == 'after_commit':
os._exit(92)
def main():
# Linux worker: bounded address space as well as the container memory limit.
import resource
memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
otel.setup(child=True)
engine = create_db_engine()
try:
with otel.span('image.process', context=otel.propagator.extract({
'traceparent': os.getenv('TRACEPARENT', ''), 'tracestate': os.getenv('TRACESTATE', '')})):
process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2])
otel.event('image_completed')
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError):
return 20
except MemoryError:
return 21
except Exception:
return 22 # No credentials or decoder/database internals in logs.
finally:
engine.dispose()
otel.shutdown()
return 0
if __name__ == '__main__':
sys.exit(main())
+199
View File
@@ -0,0 +1,199 @@
"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state."""
import argparse
import json
import logging
import os
import random
import time
from datetime import datetime, timedelta, timezone
from uuid import UUID, uuid4
from redis import Redis
from redis.exceptions import ResponseError
from sqlalchemy import and_, or_, select, func
from sqlalchemy.orm import sessionmaker
from .database import create_db_engine
from .admission import reserve_capacity
from .models import Attachment, AttachmentJob
from .object_storage import BUCKET, s3_client
from . import telemetry as otel
STREAM = 'attachment-jobs'
GROUP = 'image-workers'
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60')))
LEASE_SECONDS = TIMEOUT + 15
def now():
return datetime.now(timezone.utc)
def broker():
return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
decode_responses=True, socket_connect_timeout=2, socket_timeout=5)
def ensure_group(client):
try:
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except ResponseError as exc:
if not str(exc).startswith('BUSYGROUP'):
raise
def envelope(job):
return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1',
'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version,
'attempt': job.attempt + 1,
'created_at': job.created_at.isoformat().replace('+00:00', 'Z')}
def fail(session, job, attachment, code, permanent=False):
job.failures += 1
job.last_error = code
job.lease_token = job.lease_until = job.published_at = None
attachment.updated_at = now()
if permanent or job.failures >= MAX_ATTEMPTS:
attachment.status = 'failed'
attachment.error_code = code if permanent else 'retry_exhausted'
else:
attachment.status = 'queued'
attachment.error_code = None
delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5)
job.available_at = now() + timedelta(seconds=delay)
otel.event('job_retry' if attachment.status == 'queued' else 'job_failed',
attempt=job.attempt, outcome=attachment.status, error_code=code)
def dispatch_once(sessions, client):
# A crash after XADD but before COMMIT deliberately results in redelivery.
with sessions.begin() as session:
jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where(
or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)),
and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())),
AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at)
.with_for_update(skip_locked=True, of=AttachmentJob).limit(50)))
for job in jobs:
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status == 'processing':
if job.lease_until <= now():
fail(session, job, attachment, 'lease_expired')
continue
if job.published_at is None:
with otel.span('attachment-jobs publish', kind=otel.SpanKind.PRODUCER,
context=otel.propagator.extract(job.trace_context), attributes={
'messaging.system': 'redis', 'messaging.destination.name': STREAM,
'messaging.operation.type': 'publish', 'job.attempt': job.attempt + 1}):
client.xadd(STREAM, {'job': json.dumps({**envelope(job), **otel.carrier()})})
otel.event('job_published', attempt=job.attempt + 1)
job.published_at = now()
def sample_queue(sessions):
# Database is the durable backlog: includes outbox rows not yet in Redis and retries.
with sessions() as session:
depth, oldest = session.execute(select(func.count(), func.min(Attachment.created_at))
.select_from(Attachment).where(Attachment.status == 'queued')).one()
otel.queue_state.update(depth=depth, age=max(0, (now() - oldest).total_seconds()) if oldest else 0,
updated=time.time())
def claim(sessions, message):
if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1'
or message.get('pipeline_version') != 1):
raise ValueError('unsupported job envelope')
job_id = str(UUID(message['job_id']))
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job is None or job.attachment_id != message.get('attachment_id'):
raise ValueError('unknown job')
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status != 'queued' or job.available_at > now():
return None
job.attempt += 1
if job.first_started_at is None:
job.first_started_at = now()
job.lease_token = str(uuid4())
job.lease_until = now() + timedelta(seconds=LEASE_SECONDS)
attachment.status = 'processing'
attachment.updated_at = now()
return job.id, job.lease_token
def record_failure(sessions, job_id, token, code, permanent=False):
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job.lease_token == token:
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
fail(session, job, attachment, code, permanent)
def replay(sessions, job_id, reason):
if not reason.strip() or len(reason) > 200:
raise ValueError('Give a nonempty reason of at most 200 characters')
with sessions.begin() as session:
job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True)
if job is None:
raise ValueError('Job not found')
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status != 'failed':
raise ValueError('Replay is allowed only for failed jobs')
if not attachment.staging_key:
raise ValueError('Staging retention expired; upload the file again')
s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key)
reserve_capacity(session)
job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason,
'after_attempt': job.attempt,
'last_error': job.last_error}]
job.failures = 0
job.available_at = now()
job.published_at = job.lease_until = job.lease_token = None
attachment.status, attachment.error_code = 'queued', None
attachment.updated_at = now()
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='command', required=True)
sub.add_parser('dispatch')
sub.add_parser('failed')
retry = sub.add_parser('replay')
retry.add_argument('job_id')
retry.add_argument('--reason', required=True)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
otel.setup()
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
try:
if args.command == 'failed':
with sessions() as session:
for job, att in session.execute(select(AttachmentJob, Attachment).join(
Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == 'failed')):
print(json.dumps({'job_id': job.id, 'attachment_id': att.id,
'attempt': job.attempt, 'error_code': att.error_code,
'last_error': job.last_error,
'replay_available': bool(att.staging_key)}))
elif args.command == 'replay':
replay(sessions, args.job_id, args.reason)
print('Job queued for replay')
else:
client = broker()
while True:
try:
ensure_group(client)
dispatch_once(sessions, client)
sample_queue(sessions)
except Exception as exc:
otel.event('dispatch_deferred', error_type=type(exc).__name__)
time.sleep(1)
finally:
engine.dispose()
otel.shutdown()
if __name__ == '__main__':
main()
+99
View File
@@ -0,0 +1,99 @@
import asyncio
import os
from dataclasses import dataclass
from starlette.requests import Request
from .errors import error_response
from .middleware import is_upload
@dataclass(frozen=True)
class Limits:
max_users: int = 1000
max_chats: int = 1000
max_messages: int = 10000
max_text_bytes: int = 16 * 1024 * 1024
max_request_bytes: int = 64 * 1024
max_inflight: int = 32
message_rate: int = 1000
message_burst: int = 1000
@classmethod
def from_env(cls):
values = {}
for name, default in cls().__dict__.items():
value = int(os.getenv(name.upper(), str(default)))
if value < 1:
raise ValueError(f'{name.upper()} must be positive')
values[name] = value
return cls(**values)
limits = Limits.from_env()
class ResourceLimitMiddleware:
def __init__(self, app, settings=limits):
self.app = app
self.settings = settings
self.active = 0
async def __call__(self, scope, receive, send):
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
return await self.app(scope, receive, send)
request = Request(scope)
async def reject(status, code, message):
response = error_response(request, status, code, message)
if status == 503:
response.headers['Retry-After'] = '1'
await response(scope, receive, send)
if self.active >= self.settings.max_inflight:
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
self.active += 1
try:
# The upload endpoint authenticates and bounds its stream itself.
if is_upload(scope['method'], scope['path']):
return await self.app(scope, receive, send)
length = request.headers.get('content-length')
if length is not None:
try:
size = int(length)
if size < 0:
raise ValueError()
except ValueError:
return await reject(400, 'invalid_request', 'Неверный Content-Length')
if size > self.settings.max_request_bytes:
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
body = bytearray()
try:
async with asyncio.timeout(5):
while True:
message = await receive()
if message['type'] == 'http.disconnect':
return
chunk = message.get('body', b'')
if len(body) + len(chunk) > self.settings.max_request_bytes:
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
body.extend(chunk)
if not message.get('more_body', False):
break
except TimeoutError:
return await reject(503, 'unavailable', 'Истекло время чтения запроса')
delivered = False
async def buffered_receive():
nonlocal delivered
if not delivered:
delivered = True
return {'type': 'http.request', 'body': bytes(body), 'more_body': False}
return await receive()
await self.app(scope, buffered_receive, send)
finally:
self.active -= 1
+55
View File
@@ -0,0 +1,55 @@
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from starlette.exceptions import HTTPException
from contextlib import asynccontextmanager
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
from sqlalchemy.orm import sessionmaker
from redis.exceptions import RedisError
from botocore.exceptions import BotoCoreError, ClientError
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
from .database import create_db_engine
from .store import store
from .auth import sessions
from .middleware import RequestIdMiddleware
from .limits import ResourceLimitMiddleware
from .routers.health import router as health_router
from .routers.users import router as users_router
from .routers.chats import router as chats_router
from .routers.auth import router as auth_router
from .routers.attachments import router as attachments_router
from . import telemetry
@asynccontextmanager
async def lifespan(app):
telemetry.setup()
telemetry.seed_http(app.routes)
engine = create_db_engine()
try:
store.sessions = sessionmaker(engine, expire_on_commit=False)
yield
finally:
engine.dispose()
sessions.client.close()
telemetry.shutdown()
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
app.add_middleware(ResourceLimitMiddleware)
app.add_middleware(RequestIdMiddleware)
app.add_exception_handler(APIError, api_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(HTTPException, http_error_handler)
app.add_exception_handler(Exception, internal_error_handler)
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
app.add_exception_handler(error, dependency_error_handler)
app.include_router(health_router)
app.include_router(users_router)
app.include_router(chats_router)
app.include_router(auth_router)
app.include_router(attachments_router)
# Outside FastAPI's error middleware: includes generated 500 responses and full streams.
app = telemetry.HttpTelemetry(app)
+43
View File
@@ -0,0 +1,43 @@
from uuid import uuid4
from socket import gethostname
from starlette.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
import re
def is_upload(method, path):
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
INSTANCE_ID = gethostname()
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
request.state.request_id = request_id
if (request.method in {"POST", "PUT", "PATCH"}
and request.url.path.startswith("/api/")
and not is_upload(request.method, request.url.path)
and request.url.path != '/api/v1/auth/sessions'):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
response = JSONResponse(
status_code=415,
content={
"error": {
"code": "unsupported_media_type",
"message": "Ожидается Content-Type application/json",
"request_id": request_id,
}
},
)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
response = await call_next(request)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
+109
View File
@@ -0,0 +1,109 @@
from datetime import datetime
from uuid import uuid4
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
class Base(DeclarativeBase):
pass
class User(Base):
__tablename__ = 'users'
__table_args__ = (
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
username: Mapped[str] = mapped_column(Text, unique=True)
display_name: Mapped[str] = mapped_column(Text)
password_hash: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class Chat(Base):
__tablename__ = 'chats'
__table_args__ = (
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
Index('chats_order_idx', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
title: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
class ChatMember(Base):
__tablename__ = 'chat_members'
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
class Message(Base):
__tablename__ = 'messages'
__table_args__ = (
CheckConstraint('char_length(text) <= 4000'),
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
text: Mapped[str] = mapped_column(Text)
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class SchemaMigration(Base):
__tablename__ = 'schema_migrations'
version: Mapped[str] = mapped_column(Text, primary_key=True)
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
class Attachment(Base):
__tablename__ = 'attachments'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
object_key: Mapped[str] = mapped_column(Text, unique=True)
filename: Mapped[str] = mapped_column(Text)
content_type: Mapped[str] = mapped_column(Text)
size_bytes: Mapped[int] = mapped_column(BigInteger)
sha256: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
status: Mapped[str] = mapped_column(Text, server_default='queued')
staging_key: Mapped[str | None] = mapped_column(Text)
variants: Mapped[list] = mapped_column(JSONB, server_default='[]')
error_code: Mapped[str | None] = mapped_column(Text)
class AttachmentJob(Base):
"""The job row is also the transactional outbox and failed-job registry."""
__tablename__ = 'attachment_jobs'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True)
pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1')
attempt: Mapped[int] = mapped_column(BigInteger, server_default='0')
failures: Mapped[int] = mapped_column(BigInteger, server_default='0')
last_error: Mapped[str | None] = mapped_column(Text)
first_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]')
trace_context: Mapped[dict] = mapped_column(JSONB, server_default='{}')
class IdempotencyRecord(Base):
__tablename__ = 'message_idempotency'
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
key: Mapped[str] = mapped_column(Text, primary_key=True)
message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE'))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
+33
View File
@@ -0,0 +1,33 @@
import os
from functools import lru_cache
import boto3
from botocore.config import Config
from .telemetry import dependency_span
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
PREFIX = 'originals/'
@lru_cache(maxsize=1)
def s3_client():
client = boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
config=Config(connect_timeout=2, read_timeout=5,
retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'},
request_checksum_calculation='when_required',
response_checksum_validation='when_required'),
)
# Instrument SDK operations without object keys, signed URLs, headers or bytes.
original = client._make_api_call
def call(operation, parameters):
with dependency_span('s3.' + operation, **{'rpc.system': 'aws-api', 'rpc.service': 'S3',
'rpc.method': operation}):
return original(operation, parameters)
client._make_api_call = call
return client
View File
+90
View File
@@ -0,0 +1,90 @@
import asyncio
import hashlib
from tempfile import TemporaryFile
from urllib.parse import quote
from fastapi import APIRouter, Depends, Query, Request
from starlette.background import BackgroundTask
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
from ..dependencies import current_user
from ..errors import APIError
from ..object_storage import BUCKET, s3_client
from ..store import ensure_uuid, store
router = APIRouter(prefix='/api/v1', tags=['attachments'])
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
decode_slot = asyncio.Semaphore(1)
@router.post('/chats/{chat_id}/attachments', status_code=202)
async def upload(chat_id: str, request: Request, filename: str = Query(...),
user: dict = Depends(current_user)):
chat_id = ensure_uuid(chat_id)
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
validate_filename(filename)
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if content_type not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
try:
length = int(length)
if length < 0:
raise ValueError()
except ValueError as exc:
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
size = 0
digest = hashlib.sha256()
with TemporaryFile() as source:
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
digest.update(chunk)
await run_in_threadpool(source.write, chunk)
except (TimeoutError, ClientDisconnect) as exc:
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
if length is not None and size != length:
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
async with decode_slot:
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
filename, content_type, size, digest.hexdigest())
@router.get('/attachments/{attachment_id}')
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
return metadata(get_attachment(attachment_id, user['id']))
@router.get('/attachments/{attachment_id}/content')
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
attachment = get_attachment(attachment_id, user['id'])
if attachment.status != 'ready':
raise APIError(409, 'conflict', 'Вложение ещё не готово')
key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes
if variant != 'original':
item = next((v for v in attachment.variants if v['name'] == variant), None)
if item is None:
raise APIError(404, 'not_found', 'Вариант не найден')
key, content_type, size = item['object_key'], item['content_type'], item['size_bytes']
body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body']
def chunks():
try:
yield from body.iter_chunks(chunk_size=64 * 1024)
finally:
body.close()
return StreamingResponse(chunks(), media_type=content_type,
headers={'Content-Length': str(size),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
'X-Content-Type-Options': 'nosniff'},
background=BackgroundTask(body.close))
+34
View File
@@ -0,0 +1,34 @@
import base64
from fastapi import APIRouter, Depends, Header, Request, Response
from starlette.concurrency import run_in_threadpool
from ..auth import sessions, unauthorized
from ..dependencies import bearer_token, current_user
from ..errors import APIError
from ..store import store
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
@router.post('', status_code=201)
async def create_session(request: Request, authorization: str = Header(default='')):
if await request.body():
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
try:
scheme, encoded = authorization.split()
if scheme.lower() != 'basic':
raise ValueError()
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
raise ValueError()
except (ValueError, UnicodeError):
raise unauthorized() from None
user = await run_in_threadpool(store.login, username, password)
return await run_in_threadpool(sessions.create, user)
@router.delete('/current', status_code=204)
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
sessions.delete(token)
return Response(status_code=204)
+49
View File
@@ -0,0 +1,49 @@
from typing import Optional
import re
from fastapi import APIRouter, Depends, Header, Query, status
from ..errors import APIError
from ..dependencies import current_user, page_limit
from ..schemas.chat import Chat, ChatPage, CreateChat
from ..schemas.message import CreateMessage, Message, MessagePage
from ..store import store
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED)
def create_chat(payload: CreateChat, user: dict = Depends(current_user)):
return store.create_chat(user["id"], payload.title, payload.member_ids)
@router.get("", response_model=ChatPage)
def list_chats(
limit: int = Depends(page_limit),
cursor: Optional[str] = Query(default=None, min_length=1),
user: dict = Depends(current_user),
):
return store.list_chats(user["id"], limit, cursor)
@router.get("/{chat_id}", response_model=Chat)
def get_chat(chat_id: str, user: dict = Depends(current_user)):
return store.get_chat_for_user(chat_id, user["id"])
@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED)
def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user),
idempotency_key: Optional[str] = Header(default=None)):
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
@router.get("/{chat_id}/messages", response_model=MessagePage)
def list_messages(
chat_id: str,
limit: int = Depends(page_limit),
cursor: Optional[str] = Query(default=None, min_length=1),
user: dict = Depends(current_user),
):
return store.list_messages(chat_id, user["id"], limit, cursor)
+23
View File
@@ -0,0 +1,23 @@
from fastapi import APIRouter
from sqlalchemy import select
from ..schemas.common import Health
from ..auth import sessions
from ..store import store
from ..object_storage import BUCKET, s3_client
router = APIRouter(prefix='/health')
@router.get("/live", response_model=Health)
def health_live():
return {"status": "ok"}
@router.get("/ready", response_model=Health)
def health_ready():
with store.sessions() as session:
session.execute(select(1))
sessions.client.ping()
s3_client().head_bucket(Bucket=BUCKET)
return {"status": "ok"}
+19
View File
@@ -0,0 +1,19 @@
from fastapi import APIRouter, Depends, status
from ..dependencies import current_user
from ..schemas.user import CreateUser, User
from ..store import store
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
def create_user(payload: CreateUser):
return store.create_user(payload.username, payload.display_name, payload.password)
@router.get("/me", response_model=User)
def get_current_user(user: dict = Depends(current_user)):
return user
View File
+35
View File
@@ -0,0 +1,35 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator
from .common import ContractModel
class CreateChat(ContractModel):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: List[StrictStr] = Field(min_length=1, max_length=100)
@field_validator("title")
def validate_title(cls, value):
if not value.strip():
raise ValueError("blank title")
return value
@field_validator("member_ids")
def validate_member_ids(cls, value):
if len(value) != len(set(value)):
raise ValueError("duplicate member_ids")
return value
class Chat(ContractModel):
id: str
title: str
member_ids: List[str]
created_at: str
class ChatPage(ContractModel):
items: List[Chat]
next_cursor: Optional[str]
+17
View File
@@ -0,0 +1,17 @@
from pydantic import BaseModel, ConfigDict, field_validator
class ContractModel(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator('*')
def validate_text_encoding(cls, value, info):
if isinstance(value, str):
value.encode('utf-8')
if info.field_name != 'password' and '\x00' in value:
raise ValueError('NUL is not supported in PostgreSQL text')
return value
class Health(ContractModel):
status: str
+39
View File
@@ -0,0 +1,39 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator, model_validator
from uuid import UUID
from .common import ContractModel
class CreateMessage(ContractModel):
text: StrictStr = Field(max_length=4000)
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
def validate_ids(cls, value):
ids = [str(UUID(item)) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('duplicate attachments')
return ids
@model_validator(mode='after')
def validate_content(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('empty message')
return self
class Message(ContractModel):
id: str
chat_id: str
sender_id: str
text: str
attachment_ids: List[str]
created_at: str
class MessagePage(ContractModel):
items: List[Message]
next_cursor: Optional[str]
+34
View File
@@ -0,0 +1,34 @@
import re
from pydantic import Field, StrictStr, field_validator
from .common import ContractModel
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class CreateUser(ContractModel):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
def validate_username(cls, value):
if not USERNAME_RE.fullmatch(value):
raise ValueError("invalid username")
return value
@field_validator("display_name")
def validate_display_name(cls, value):
if not value.strip():
raise ValueError("blank display_name")
return value
class User(ContractModel):
id: str
username: str
display_name: str = Field(min_length=1, max_length=100)
created_at: str
+229
View File
@@ -0,0 +1,229 @@
import base64
import json
from datetime import datetime, timezone
from typing import Tuple
from uuid import UUID
from sqlalchemy import select, tuple_, update
from sqlalchemy.exc import IntegrityError
from .auth import password_hasher, verify_password, unauthorized
from .errors import APIError
from .cache import history_cache
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
def ensure_uuid(value):
try:
if not isinstance(value, str):
raise ValueError()
return str(UUID(value))
except (TypeError, ValueError) as exc:
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
def serialize(model, *fields):
if model is None:
return None
result = {}
for field in fields:
value = getattr(model, field)
if isinstance(value, datetime):
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
result[field] = value
return result
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
CHAT_FIELDS = ('id', 'title', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
def after_condition(model, after):
return tuple_(model.created_at, model.id) > (
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
def history_query(chat_id, limit, after):
query = select(Message).where(Message.chat_id == chat_id)
if after:
query = query.where(after_condition(Message, after))
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
class PostgresStore:
def __init__(self, cache=history_cache):
self.sessions = None
self.cache = cache
def create_user(self, username, display_name, password):
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
try:
with self.sessions.begin() as session:
session.add(user)
session.flush()
result = serialize(user, *USER_FIELDS)
return result
except IntegrityError as exc:
if getattr(exc.orig, 'sqlstate', None) == '23505':
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
raise
def get_user(self, user_id):
with self.sessions() as session:
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
def login(self, username, password):
with self.sessions() as session:
user = session.scalar(select(User).where(User.username == username))
encoded = user.password_hash if user else None
result = serialize(user, *USER_FIELDS)
if not verify_password(encoded, password):
raise unauthorized()
return result
def create_chat(self, actor_id, title, member_ids):
members = [ensure_uuid(member) for member in member_ids]
if len(members) != len(set(members)) or actor_id not in members:
raise APIError(400, 'invalid_request', 'Неверный список участников')
try:
with self.sessions.begin() as session:
chat = Chat(title=title)
session.add(chat)
session.flush()
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
result = serialize(chat, *CHAT_FIELDS)
result['member_ids'] = sorted(members)
return result
except IntegrityError as exc:
if getattr(exc.orig, 'sqlstate', None) == '23503':
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
raise
@staticmethod
def _chat(session, chat_id, actor_id):
chat = session.scalar(select(Chat).join(ChatMember).where(
Chat.id == chat_id, ChatMember.user_id == actor_id))
if chat is None:
raise APIError(404, 'not_found', 'Объект не найден')
return chat
@staticmethod
def _members(session, chat):
result = serialize(chat, *CHAT_FIELDS)
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
return result
def get_chat_for_user(self, chat_id, actor_id):
with self.sessions() as session:
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
def list_chats(self, actor_id, limit, cursor):
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
if after:
query = query.where(after_condition(Chat, after))
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
with self.sessions() as session:
chats = list(session.scalars(query))
items = [self._members(session, chat) for chat in chats[:limit]]
return self._page(items, len(chats) > limit, 'chats', actor_id)
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
chat_id = ensure_uuid(chat_id)
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
with self.sessions.begin() as session:
self._chat(session, chat_id, actor_id)
if attachment_ids:
found = list(session.scalars(select(Attachment).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)
.order_by(Attachment.id).with_for_update()))
if len(found) != len(attachment_ids):
raise APIError(404, 'not_found', 'Объект не найден')
if idempotency_key is not None:
# Serialize writers to this chat before looking up the key.
# The same row is updated below for history cache invalidation.
session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update())
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
if record is not None:
previous = session.get(Message, record.message_id)
if previous.text != text or previous.attachment_ids != attachment_ids:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
return serialize(previous, *MESSAGE_FIELDS)
if attachment_ids and any(item.status == 'failed' for item in found):
raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой')
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
session.add(message)
session.flush()
if idempotency_key is not None:
session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id,
key=idempotency_key, message_id=message.id))
result = serialize(message, *MESSAGE_FIELDS)
return result
def list_messages(self, chat_id, actor_id, limit, cursor):
chat_id = ensure_uuid(chat_id)
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
with self.sessions() as session:
version = self._chat(session, chat_id, actor_id).history_version
return self.cache.get_or_load(
chat_id, version, limit, after,
lambda: self._load_messages(chat_id, limit, after),
)
def _load_messages(self, chat_id, limit, after):
with self.sessions() as session:
messages = list(session.scalars(history_query(chat_id, limit, after)))
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
return self._page(items, len(messages) > limit, 'messages', chat_id)
def _page(self, items, has_more, resource, owner_id):
cursor = None
if has_more:
last = items[-1]
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
return {'items': items, 'next_cursor': cursor}
@staticmethod
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
payload = {
"resource": resource,
"owner_id": owner_id,
"created_at": created_at,
"id": item_id,
}
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
@staticmethod
def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]:
try:
padded = cursor + "=" * (-len(cursor) % 4)
payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
raise APIError(400, "invalid_request", "Неверный cursor") from exc
if not isinstance(payload, dict):
raise APIError(400, "invalid_request", "Неверный cursor")
if payload.get("resource") != resource or payload.get("owner_id") != owner_id:
raise APIError(400, "invalid_request", "Неверный cursor")
item_id = ensure_uuid(payload.get("id"))
created_at = payload.get("created_at")
if not isinstance(created_at, str) or not created_at:
raise APIError(400, "invalid_request", "Неверный cursor")
try:
parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc:
raise ValueError("Expected UTC timestamp")
created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z")
except ValueError as exc:
raise APIError(400, "invalid_request", "Неверный cursor") from exc
return created_at, item_id
store = PostgresStore()
+206
View File
@@ -0,0 +1,206 @@
"""Small, explicit OTel instrumentation. Only allowlisted technical fields leave the app."""
import json
import logging
import os
import re
import socket
import time
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
import psutil
from opentelemetry import metrics, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider, Histogram
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
tracer = trace.get_tracer('messenger', '1.6.0')
meter = metrics.get_meter('messenger', '1.6.0')
propagator = TraceContextTextMapPropagator() # No baggage: it may contain user data.
logger = logging.getLogger('messenger.events')
logger.propagate = False
logger.setLevel(logging.INFO)
providers = []
LOG_FIELDS = {'method', 'route', 'status_class', 'status', 'duration_seconds', 'request_id',
'operation', 'error_type', 'attempt', 'outcome', 'error_code', 'job_type'}
http_requests = meter.create_counter('http_requests')
http_duration = meter.create_histogram('http_duration', unit='s')
jobs_processed = meter.create_counter('jobs_processed')
job_duration = meter.create_histogram('job_duration', unit='s')
time_to_ready = meter.create_histogram('attachment_time_to_ready', unit='s')
worker_busy = meter.create_up_down_counter('worker_busy')
queue_state = {'depth': 0, 'age': 0, 'updated': 0}
class JsonFormatter(logging.Formatter):
def format(self, record):
context = trace.get_current_span().get_span_context()
return json.dumps({
'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
'severity': record.levelname, 'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'),
'event': record.getMessage(),
'trace_id': format(context.trace_id, '032x') if context.is_valid else '',
'span_id': format(context.span_id, '016x') if context.is_valid else '',
**{key: record.__dict__[key] for key in LOG_FIELDS if key in record.__dict__},
})
def event(name, *, severity=logging.INFO, **fields):
# Call sites pass fixed event names, safe enums/numbers and technical correlation IDs.
logger.log(severity, name, extra={'event': name, **{k: v for k, v in fields.items() if k in LOG_FIELDS}})
def setup(*, child=False):
if providers or os.getenv('OTEL_SDK_DISABLED', 'false').lower() == 'true':
return
resource = Resource({
'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'),
'service.version': '1.6.0', 'service.instance.id': socket.gethostname(),
'deployment.environment.name': 'lab',
})
endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
traces = TracerProvider(resource=resource)
traces.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + '/v1/traces', timeout=1),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000))
trace.set_tracer_provider(traces)
logs = LoggerProvider(resource=resource)
logs.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + '/v1/logs', timeout=1),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000))
console = logging.StreamHandler()
console.setFormatter(JsonFormatter())
logger.addHandler(console)
logger.addHandler(LoggingHandler(logger_provider=logs))
providers.extend([traces, logs])
if child:
return # Short-lived image processes emit spans/logs; parent owns worker metrics.
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + '/v1/metrics', timeout=1),
export_interval_millis=5000, export_timeout_millis=2000)
buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 30, 60, 120, 300]
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
View(instrument_type=Histogram,
aggregation=ExplicitBucketHistogramAggregation(buckets))])
metrics.set_meter_provider(metric_provider)
providers.append(metric_provider)
process = psutil.Process()
meter.create_observable_gauge('process_memory', unit='By', callbacks=[
lambda _: [metrics.Observation(process.memory_info().rss)]])
meter.create_observable_counter('process_cpu', unit='s', callbacks=[
lambda _: [metrics.Observation(sum(process.cpu_times()[:2]))]])
cgroup = Path('/sys/fs/cgroup')
if (cgroup / 'memory.current').exists():
meter.create_observable_gauge('container_memory', unit='By', callbacks=[
lambda _: [metrics.Observation(int((cgroup / 'memory.current').read_text()))]])
def cpu_usage(_):
values = dict(line.split() for line in (cgroup / 'cpu.stat').read_text().splitlines())
return [metrics.Observation(int(values['usage_usec']) / 1_000_000)]
meter.create_observable_counter('container_cpu', unit='s', callbacks=[cpu_usage])
if os.getenv('OTEL_SERVICE_NAME') == 'messenger-worker':
capacity = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
meter.create_observable_gauge('worker_capacity', callbacks=[
lambda _: [metrics.Observation(capacity)]])
worker_busy.add(0)
if os.getenv('OTEL_SERVICE_NAME') == 'messenger-dispatcher':
for name, key, unit in [('queue_depth', 'depth', ''), ('queue_oldest_age', 'age', 's'),
('queue_sample_timestamp', 'updated', 's')]:
meter.create_observable_gauge(name, unit=unit, callbacks=[
lambda _, key=key: [metrics.Observation(queue_state[key], {'queue': 'attachment-jobs'})]])
def shutdown():
for provider in reversed(providers):
provider.shutdown()
providers.clear()
def seed_http(routes):
# Export zero before the first failure: Prometheus rate needs a prior sample.
for route in routes:
if route.path.startswith('/health/'):
continue
for method in route.methods or ():
for status in ('2xx', '3xx', '4xx', '5xx'):
http_requests.add(0, {'method': method, 'route': route.path, 'status_class': status})
def carrier():
result = {}
propagator.inject(result)
return result
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, context=None, attributes=None):
# Never export exception messages/stack traces (SQL errors can embed parameters).
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attributes,
record_exception=False, set_status_on_exception=False) as current:
try:
yield current
except Exception as exc:
current.set_status(Status(StatusCode.ERROR))
current.set_attribute('error.type', type(exc).__name__)
event('operation_failed', severity=logging.ERROR, operation=name, error_type=type(exc).__name__)
raise
@contextmanager
def dependency_span(name, **attributes):
# Background health probes and queue polling must not create disconnected trace noise.
if not trace.get_current_span().get_span_context().is_valid:
yield
else:
with span(name, kind=SpanKind.CLIENT, attributes=attributes):
yield
class HttpTelemetry:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
return await self.app(scope, receive, send)
headers = {k.decode('latin1'): v.decode('latin1') for k, v in scope['headers']
if k in (b'traceparent', b'tracestate')}
method = scope['method'] if scope['method'] in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
status = 500
started = time.monotonic()
with span('HTTP ' + method, kind=SpanKind.SERVER, context=propagator.extract(headers)) as current:
async def response(message):
nonlocal status
if message['type'] == 'http.response.start':
status = message['status']
await send(message)
try:
await self.app(scope, receive, response)
finally:
route = getattr(scope.get('route'), 'path', 'unmatched')
labels = {'method': method, 'route': route, 'status_class': f'{status // 100}xx'}
elapsed = time.monotonic() - started
http_requests.add(1, labels)
http_duration.record(elapsed, labels)
current.update_name(method + ' ' + route)
current.set_attributes({'http.request.method': method, 'http.route': route,
'http.response.status_code': status})
if status >= 500:
current.set_status(Status(StatusCode.ERROR))
request_id = scope.get('state', {}).get('request_id', '')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', request_id):
request_id = ''
current.set_attribute('request_id', request_id)
event('http_request', severity=logging.ERROR if status >= 500 else logging.INFO,
**labels, status=status, duration_seconds=elapsed,
request_id=request_id)
+122
View File
@@ -0,0 +1,122 @@
"""Bounded consumers, one unacknowledged message per consumer, hard process timeout."""
import json
import logging
import os
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from sqlalchemy.orm import sessionmaker
from .database import create_db_engine
from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure
from .models import Attachment, AttachmentJob
from . import telemetry as otel
def handle(sessions, client, message_id, fields):
try:
message = json.loads(fields['job'])
if not isinstance(message, dict):
raise ValueError('invalid envelope')
except (ValueError, KeyError, TypeError):
otel.event('invalid_job')
acknowledge(client, message_id)
return
with otel.span('attachment-jobs process', kind=otel.SpanKind.CONSUMER,
context=otel.propagator.extract(message), attributes={
'messaging.system': 'redis', 'messaging.destination.name': STREAM,
'messaging.operation.type': 'process', 'job.attempt': message.get('attempt', 0)}):
process_message(sessions, client, message_id, message)
def process_message(sessions, client, message_id, message):
try:
claimed = claim(sessions, message)
except (ValueError, KeyError, TypeError):
otel.event('invalid_job')
claimed = None
if claimed:
job_id, token = claimed
started = time.monotonic()
outcome = 'failed'
otel.worker_busy.add(1)
try:
environment = {**os.environ, 'OTEL_CHILD_PROCESS': 'true'}
environment.update({key.upper(): value for key, value in otel.carrier().items()})
try:
result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token],
timeout=TIMEOUT, check=False, env=environment)
code = result.returncode
except subprocess.TimeoutExpired:
code = 23
if code in (91, 92):
# Simulate a lost worker before ack: leave the pending entry and lease.
return
if code:
record_failure(sessions, job_id, token,
{20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'}
.get(code, 'processing_failed'), permanent=code in (20, 21))
otel.trace.get_current_span().set_status(otel.Status(otel.StatusCode.ERROR))
else:
with sessions() as session:
job = session.get(AttachmentJob, job_id)
attachment = session.get(Attachment, job.attachment_id)
outcome = 'ready' if attachment.status == 'ready' else 'skipped'
if outcome == 'ready':
otel.time_to_ready.record((attachment.updated_at - attachment.created_at).total_seconds(),
{'job_type': 'attachment.process.v1'})
finally:
otel.worker_busy.add(-1)
labels = {'job_type': 'attachment.process.v1', 'outcome': outcome}
elapsed = time.monotonic() - started
otel.jobs_processed.add(1, labels)
otel.job_duration.record(elapsed, labels)
otel.event('job_processed', severity=logging.ERROR if outcome == 'failed' else logging.INFO,
**labels, duration_seconds=elapsed)
# Only after a durable ready/failed/retry (or an existing durable lease).
acknowledge(client, message_id)
def acknowledge(client, message_id):
with client.pipeline(transaction=True) as pipe:
pipe.xack(STREAM, GROUP, message_id)
pipe.xdel(STREAM, message_id)
pipe.execute()
def consume(sessions):
client = broker()
consumer = str(uuid4())
cursor = '0-0'
while True:
try:
ensure_group(client)
pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000,
start_id=cursor, count=1)
cursor, messages = pending[0], pending[1]
if not messages:
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
messages = batches[0][1] if batches else []
for message_id, fields in messages:
handle(sessions, client, message_id, fields)
except Exception as exc:
otel.event('worker_deferred', error_type=type(exc).__name__)
time.sleep(1)
def main():
logging.basicConfig(level=logging.INFO)
otel.setup()
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
with ThreadPoolExecutor(max_workers=concurrency) as pool:
for task in [pool.submit(consume, sessions) for _ in range(concurrency)]:
task.result()
if __name__ == '__main__':
main()
+290
View File
@@ -0,0 +1,290 @@
"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage.
Uses only the standard library. Stops services only in this Compose project and
always starts them again; never deletes volumes. Run separately from other tests.
"""
import argparse
import json
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
PROM = 'http://127.0.0.1:9090'
LOKI = 'http://127.0.0.1:3100'
TEMPO = 'http://127.0.0.1:3200'
def stamp():
return datetime.now(timezone.utc).isoformat()
def get(url, **params):
if params:
url += '?' + urllib.parse.urlencode(params)
with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r:
return json.load(r)
def query(expr):
data = get(PROM + '/api/v1/query', query=expr)
check(data['status'] == 'success', 'PromQL failed')
return data['data']['result']
def number(expr):
values = query(expr)
return float(values[0]['value'][1]) if values else 0.0
def wait(label, fn, timeout=90):
end = time.monotonic() + timeout
last = None
while time.monotonic() < end:
try:
value = fn()
if value:
return value
except (urllib.error.URLError, TimeoutError, AssertionError) as exc:
last = type(exc).__name__
time.sleep(2)
raise AssertionError(f'Timeout: {label} ({last})')
def compose(*args):
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90)
def trace_data(trace_id):
data = get(TEMPO + '/api/traces/' + trace_id)
spans = []
for batch in data.get('batches', data.get('resourceSpans', [])):
resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])}
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
for span in scope.get('spans', []):
spans.append({**span, 'service': resource.get('service.name')})
return data, spans
def logs(trace_id):
return get(LOKI + '/loki/api/v1/query_range',
query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"',
start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result']
def attrs(span):
return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])}
def pipeline(trace_id, *, retry=False):
data, spans = trace_data(trace_id)
names = {s['name'] for s in spans}
required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish',
'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'}
if not required <= names:
return None
by_id = {s['spanId']: s for s in spans}
for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'),
('attachment-jobs process', 'attachment-jobs publish'),
('image.process', 'attachment-jobs process')]:
for s in (s for s in spans if s['name'] == name):
check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name,
'Broken trace parent for ' + name)
check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name)
attempts = sorted({int(attrs(s)['job.attempt']) for s in spans
if s['name'] == 'attachment-jobs process'})
if retry and len(attempts) < 2:
return None
records = logs(trace_id)
services = {r['stream'].get('service_name') for r in records}
if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services:
return None
return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names),
'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True,
'tracestate_preserved': True,
'_raw': json.dumps([data, records])}
class Scenario:
def __init__(self, base_url):
self.client = Client(base_url)
self.alice, _, _, self.chat = self.client.bootstrap()
self.private_markers = [self.alice['token'], self.alice['registration']['password']]
def upload(self):
trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4())
filename = 'private_filename_' + uuid.uuid4().hex + '.png'
self.private_markers.append(filename)
result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename,
202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01',
'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment')
check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed')
return result['json'], trace_id, request_id
def check_pipeline(self, trace_id, request_id, retry=False):
result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry))
raw = result.pop('_raw')
check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry')
check(request_id in raw, 'Request ID is missing from trace/log correlation')
result['request_id'] = request_id
result['private_inputs_absent'] = True
return result
def business(self, expected=200, trace_id=None):
headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {}
return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected,
actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None)
def smoke(self):
att, trace_id, request_id = self.upload()
self.client.wait_ready(self.alice, att)
result = self.check_pipeline(trace_id, request_id)
names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total',
'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket',
'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting',
'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes']
for name in names:
wait(name, lambda name=name: query(name))
series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data']
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'}
check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label')
check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics')
result['metrics_present'] = names
return result
def queue(self):
result = {'started_at': stamp()}
compose('stop', 'worker')
try:
att, trace_id, request_id = self.upload()
self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']])
wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1)
result.update(firing_at=stamp(), depth=number('max(queue_depth)'),
oldest_age_seconds=number('max(queue_oldest_age_seconds)'))
finally:
compose('start', 'worker')
ready = self.client.wait_ready(self.alice, att)
wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}'))
result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'),
pipeline=self.check_pipeline(trace_id, request_id))
return result
def database(self):
result = {'started_at': stamp()}
trace_id = uuid.uuid4().hex
compose('stop', 'postgres')
try:
errors = 0
def load_until_firing():
nonlocal errors
self.business(expected=503, trace_id=trace_id if errors == 0 else None)
errors += 1
return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1
wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120)
result['failed_requests'] = errors
result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m'))
def failed_trace():
_, spans = trace_data(trace_id)
failed = [s for s in spans if s['name'].startswith('db.')
and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')]
return [{'name': s['name'], 'error_type': attrs(s).get('error.type'),
'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9}
for s in failed]
result['db_error_spans'] = wait('failed DB span', failed_trace)
result['trace_id'] = trace_id
result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id)))
finally:
compose('start', 'postgres')
wait('API recovered', self.business)
for _ in range(10):
self.business()
wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160)
result['resolved_at'] = stamp()
return result
def retry(self):
failures = number('sum(jobs_processed_total{outcome="failed"})')
compose('stop', 'worker')
try:
att, trace_id, request_id = self.upload()
compose('stop', 's3')
# `compose start worker` can also start its dependency graph, including S3.
compose('up', '-d', '--no-deps', '--no-recreate', 'worker')
wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40)
finally:
compose('start', 's3', 'worker')
wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health'))
self.client.wait_ready(self.alice, att)
return self.check_pipeline(trace_id, request_id, retry=True)
def backend(self):
accepted = 'sum(otelcol_receiver_accepted_spans_total)'
sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})'
queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
# A quiet interval on both ends makes accepted-minus-sent a loss count,
# not a count of failed export attempts or currently buffered spans.
time.sleep(8)
wait('trace queue drained', lambda: number(queue) == 0)
before = {'accepted': number(accepted), 'sent': number(sent)}
result = {'started_at': stamp(), 'before': before}
samples, writes = [], 0
compose('stop', 'tempo')
try:
end = time.monotonic() + 60
att, _, _ = self.upload()
self.client.wait_ready(self.alice, att)
while time.monotonic() < end:
body = 'private_message_' + uuid.uuid4().hex
self.private_markers.append(body)
self.client.message(self.alice, self.chat, body)
writes += 1
samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'),
'queue_batches': number(queue)})
time.sleep(0.3)
check(writes > 10, 'Not enough writes during telemetry outage')
check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded')
check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024,
'Collector exceeded its memory budget')
finally:
compose('start', 'tempo')
wait('trace queue drained', lambda: number(queue) == 0)
time.sleep(10) # Include final Prometheus scrape after drain.
after = {'accepted': number(accepted), 'sent': number(sent)}
lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent'])
check(lost >= 0, 'Loss window was not quiescent; rerun backend separately')
result.update(after=after, lost_accepted_spans=lost, successful_writes=writes,
attachment_ready_during_outage=True, samples=samples, recovered_at=stamp())
att, trace_id, request_id = self.upload()
self.client.wait_ready(self.alice, att)
result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke')
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
args = parser.parse_args()
scenario = Scenario(args.base_url)
path = ROOT / 'evidence/observability.json'
evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]):
print('Running ' + name, flush=True)
evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()}
path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
print('PASS: ' + name, flush=True)
if __name__ == '__main__':
main()