ЛР 6 — БВТ2301 — Курлова — на 5 #18

Closed
alenccha wants to merge 1 commits from alenccha/mtusi2026lab6:solution/bvt2301-kurlova into main
86 changed files with 10512 additions and 21 deletions
+43 -7
View File
@@ -1,30 +1,66 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Values below are the ones used for evidence/ (see REPORT.md for the mapping).
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
GRACEFUL_TIMEOUT=8
MAX_INFLIGHT_REQUESTS=64
MAX_BODY_BYTES=65536
MAX_RSS_MB=0
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_TIMEOUT=3
DB_CONNECT_TIMEOUT=3
DB_STATEMENT_TIMEOUT_MS=5000
DB_OP_TIMEOUT=6
REQUEST_TIMEOUT=30
DB_READY_TIMEOUT=1.5
READY_CHECK_SECONDS=1
READY_FAILURES=2
SESSION_CACHE_TTL_SECONDS=30
SESSION_SWEEP_SECONDS=60
CACHE_TIMEOUT=0.2
SESSION_REVOKED_TTL_SECONDS=60
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
UPLOAD_CONCURRENCY=2
RECONCILE_INTERVAL_SECONDS=300
RECONCILE_GRACE_SECONDS=900
# --- lab5: broker, jobs, worker
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
WORKER_PREFETCH=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=20
JOB_BACKOFF_BASE_SECONDS=2
JOB_BACKOFF_MAX_SECONDS=30
JOB_SWEEP_SECONDS=5
WORKER_CHILD_MEMORY_MB=1024
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing.
MAX_PENDING_JOBS=200
JPEG_QUALITY=85
WEBP_QUALITY=80
# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else.
FAULT_INJECTION=1
# --- lab6: OpenTelemetry. Compose sets the OTEL_* variables for api/worker (x-otel-env); shell values override:
# service.name is fixed in code: messenger-api / messenger-worker.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
# Track 1 knobs (default = telemetry on, 100% head sampling, collector profile "default"):
# OTEL_SDK_DISABLED=false
# OTEL_TRACES_SAMPLER_ARG=1.0
# COLLECTOR_PROFILE=default # or "tail" (balanced tail sampling)
+36
View File
@@ -0,0 +1,36 @@
name: Application stand
# Student job: builds the application, starts the whole Compose stand and runs the public smoke and the
# observability check. Credentials are random, generated inside the job, and thrown away with it.
# Triggers deliberately exclude pull_request_target: code from a fork never gets repository secrets.
on: [push, pull_request]
permissions:
contents: read
jobs:
stand:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Create local .env and random credentials
run: sh scripts/init-secrets.sh
- name: Validate Compose configuration
run: docker compose --env-file .env config --quiet
- name: Build and start the stand, wait for readiness
run: docker compose --env-file .env up -d --build --wait --wait-timeout 300
- name: Contract structure
run: make check
- name: Public API smoke with telemetry enabled
run: make test
- name: Traces, logs, metrics correlation and privacy
run: python3 student-tests/test_observability.py
- name: Service state and application logs on failure
if: failure()
run: |
docker compose ps -a
docker compose logs --tail 80 api-1 api-2 worker otel-collector tempo loki prometheus
- name: Remove the stand
if: always()
run: docker compose down -v
+35
View File
@@ -0,0 +1,35 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=120 \
PIP_RETRIES=10
RUN apt-get update \
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-deps -r requirements.txt
COPY app/ app/
COPY migrations/ migrations/
COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
STOPSIGNAL SIGTERM
CMD ["sh", "-c", "exec uvicorn app.main:app --no-access-log --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""]
+289 -14
View File
@@ -1,37 +1,312 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Алёна Курлова, БВТ2301
- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена наблюдаемости»**
- Commit/PR: TODO после публикации (ветка `solution/bvt2301-kurlova`)
- Источники, библиотеки, помощь ИИ: OpenTelemetry Python SDK 1.30.0 / instrumentation 0.51b0 (FastAPI, botocore), OpenTelemetry Collector contrib 0.120.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.1, Grafana 11.5.2; документация [Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/), [W3C Trace Context](https://www.w3.org/TR/trace-context/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), [tail sampling processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor). Код lab5 перенесён без изменения API. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; все числа ниже получены на локальном стенде и сохранены в `evidence/`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Desktop (Compose v2.20+, нужен `--wait`), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7.2, Docker VM: 10 CPU / 7.7 GiB.
```sh
sh scripts/init-secrets.sh # .env и случайные credentials; существующие файлы не перезаписываются
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait # 15 сервисов, включая collector/prometheus/loki/tempo/grafana
make check && make test
```
Bootstrap выполняется автоматически при каждом `up` и безопасен при повторе: `migrate` (SQL-миграции с checksum и advisory lock), `minio-init` (bucket, политика, пользователь приложения), топология RabbitMQ объявляется кодом. **Lab6:** Grafana при старте импортирует datasources (`infra/grafana/provisioning/datasources`) и dashboard (`infra/grafana/dashboards/messenger.json`, папка «Messenger»), Prometheus читает правила из `infra/prometheus/rules/`. Ручного набора dashboard нет.
Интерфейсы (только `127.0.0.1`): API `:8080` (nginx), Grafana `:3000` (admin / `secrets/grafana_admin_password`), query API Prometheus `:9090`, Loki `:3100`, Tempo `:3200` — нужны скриптам доказательств. OTLP-порты collector (4317/4318) и его метрики (8888) наружу не публикуются.
Проверки (стенд запущен, `FAULT_INJECTION=1` только на учебном стенде):
```sh
make test # публичный smoke при включённой телеметрии
python3 student-tests/test_observability.py # уровни 3-4: trace/log/metrics одного upload и одной 500, приватность
python3 student-tests/check_dashboard.py # все PromQL-запросы dashboard валидны и с данными
caffeinate -i python3 student-tests/test_incidents.py # уровень 4: инциденты A/B/C, firing -> resolved (~15 мин)
caffeinate -i python3 student-tests/test_worker_saturation.py # алерт насыщения worker (~10 мин)
caffeinate -i python3 student-tests/test_exporter_queue_full.py # очередь exporter до предела (~5 мин)
caffeinate -i python3 student-tests/track1.py run --repeats 3 # трек 5.1 (~35 мин), затем: python3 student-tests/track1.py summary
python3 student-tests/test_worker_down.py; python3 student-tests/test_failure_windows.py # регрессия lab5
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем docker compose start api-1 api-2
```
`caffeinate -i` (macOS) нужен для длинных прогонов: в первой попытке Mac ушёл в сон посреди инцидента, весь стенд «замёрз» на 18 минут (виден скачок gauge 94 → 1176 s), см. раздел «Ограничения».
Безопасная остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных (включая историю метрик/логов/трасс): `docker compose down -v`. `secrets/` и `.env` в git не попадают.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
client ─► nginx ─► api-1, api-2 ──(outbox)──► RabbitMQ ─► worker ×2
│ │ PostgreSQL / Valkey / MinIO │
│ └──── OTLP/HTTP :4318 ◄─────────────┘ (traces, metrics, logs; service.name
▼ messenger-api / messenger-worker)
otel-collector memory_limiter → attributes/scrub → [tail_sampling] → batch
├─ otlp/tempo ─► Tempo (трассы, volume tempodata)
├─ otlphttp/prometheus ─► Prometheus /api/v1/otlp (метрики, volume promdata, правила/алерты)
└─ otlphttp/loki ─► Loki /otlp (логи, volume lokidata)
Prometheus также скрейпит self-metrics collector :8888, Tempo, Loki.
Grafana (volume grafanadata) — только интерфейс и запросы к трём источникам.
```
Сеть у всех сервисов одна: внутренняя сеть Compose `default`. Порты в столбце «Порт» — внутренние; наружу опубликованы только отмеченные `127.0.0.1`. Credentials передаются Docker secrets из `secrets/` (создаёт `scripts/init-secrets.sh`), приложение читает их через переменные `*_FILE`. Digest образов указан в `compose.yaml`.
| Роль | Сервис | Версия | Health / readiness | Volume | Порт | Credentials (Docker secrets) | Назначение |
| --- | --- | --- | --- | --- | --- | --- | --- |
| API ×2 | `api-1`, `api-2` | сборка из `Dockerfile`, python:3.12.8-slim-bookworm + digest | `GET /health/ready` | нет | 8080, напрямую не опубликован | `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | REST API, outbox relay, gauges очереди |
| Балансировщик | `nginx` | nginx:1.27.3-alpine | `GET /health/live`; проверка реплик в `infra/nginx-health.sh` | нет | `127.0.0.1:8080` | нет | единая точка входа |
| Worker ×2 | `worker` | тот же образ | свежесть файла heartbeat, 15 с | нет | нет | `database_url`, `s3_access_key_id`, `s3_secret_access_key`, `broker_url` | обработка изображений |
| БД | `postgres` | postgres:17.7-alpine | `pg_isready` | `pgdata` | 5432 | `postgres_password` | источник истины |
| Кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | нет, кэш без персистентности | 6379 | нет, доступ только из внутренней сети | ускоряет проверку сессий; при сбое API читает PostgreSQL |
| S3 | `minio` | madlabcg/minio по digest | `mc ready local` | `miniodata` | 9000 | `minio_root_user`, `minio_root_password` | вложения, приватный bucket |
| Брокер | `rabbitmq` | rabbitmq:4.0.5-alpine | `check_port_connectivity` | `rabbitmqdata` | 5672 | `rabbitmq_password` | durable очередь заданий |
| Bootstrap БД | `migrate` | тот же образ | разовый, успешное завершение | нет | нет | `database_url` | SQL-миграции |
| Bootstrap S3 | `minio-init` | madlabcg/minio по digest | разовый, успешное завершение | нет | нет | `minio_root_user`, `minio_root_password`, `s3_access_key_id`, `s3_secret_access_key` | bucket, политика, пользователь приложения |
| Collector | `otel-collector` | otel/opentelemetry-collector-contrib:0.120.0 | Docker healthcheck нет: в образе нет shell; расширение `health_check` слушает 13133 | нет, это не хранилище | 4317, 4318, 8888 | нет, доступ только из внутренней сети | приём OTLP, ограничение памяти, очистка атрибутов, батчи, tail sampling, экспорт с ограниченной очередью |
| Метрики | `prometheus` | prom/prometheus:v3.2.1 | `/-/ready` | `promdata`, 15 дней | `127.0.0.1:9090` | нет, только localhost | числовые ряды: RPS, ошибки, гистограммы, gauges; SLI, recording rules и алерты |
| Логи | `loki` | grafana/loki:3.4.2 | `/ready` | `lokidata`, 7 дней | `127.0.0.1:3100` | нет, только localhost | события с текстом; индекс только по низкокардинальным labels, `event`/`trace_id`/`span_id` — structured metadata |
| Трассы | `tempo` | grafana/tempo:2.7.1 | `/ready` | `tempodata`, 7 дней | `127.0.0.1:3200` | нет, только localhost | spans с причинной связью, поиск TraceQL |
| UI | `grafana` | grafana/grafana:11.5.2 | `/api/health` | `grafanadata` | `127.0.0.1:3000` | `grafana_admin_password` | dashboard, Explore, переходы лог → трасса и трасса → логи |
Почему три хранилища: Grafana сама сигналы не хранит; метрика отвечает «сколько и как часто» (дёшево, агрегируется, но теряет конкретный запрос), лог — «что произошло в событии» (текст и технические ID), трасса — «где в цепочке процессов ушло время/возникла ошибка».
Что изменено относительно lab5 (бизнес-API и контракт не менялись):
- `app/telemetry.py` — настройка SDK (resource, sampler, views, exporters), JSON-логи в stdout с `trace_id`/`span_id`, OTLP-логи, инструменты метрик, перенос контекста;
- `app/main.py` — инструментирование FastAPI (после регистрации middleware, иначе ответы 429/503/500 самого guard не попадали в spans и метрики — найдено проверкой), `messenger.request_id` в server span, gauges очереди из PostgreSQL, тестовая инъекция `X-Fault: slow|error` (только `FAULT_INJECTION=1`);
- `app/db.py` — собственные CLIENT spans PostgreSQL и span ожидания пула. Автоинструментация `opentelemetry-instrumentation-psycopg 0.51b0` для async-курсоров закрывала span до завершения `await` (все db-spans были 0.0 ms) — с ней нельзя было отличить медленную БД от handler, поэтому она убрана;
- `app/jobs.py`, `app/outbox.py`, `app/broker.py`, `app/worker.py` — W3C trace context в сообщении задания, producer/consumer spans, метрики jobs и time-to-ready;
- `compose.yaml` — collector, Prometheus, Loki, Tempo, Grafana, volumes; `infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana` — конфиги и provisioning;
- `.github/workflows/stand.yml` — CI: сборка, запуск стенда, `make test`, проверка наблюдаемости; `scripts/init-secrets.sh` — локальные credentials;
- `docs/runbooks.md` — runbook каждого алерта; `student-tests/` — проверки lab6 (lab5-тесты сохранены).
Соответствие переменных `.env.example`: `OTEL_SERVICE_NAME` не используется — `service.name` задаётся кодом (`messenger-api`/`messenger-worker`), чтобы worker не мог унаследовать имя API из общего `.env`. `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, `OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab` — как в примере (задаются в `x-otel-env` compose). Дополнительно: `OTEL_SDK_DISABLED`, `OTEL_TRACES_SAMPLER_ARG`, `OTEL_METRIC_EXPORT_INTERVAL=10000`, `OTEL_BSP_MAX_QUEUE_SIZE=2048`, `OTEL_BLRP_MAX_QUEUE_SIZE=2048`, `OTEL_EXPORTER_OTLP_TIMEOUT=5`, `OTEL_SEMCONV_STABILITY_OPT_IN=http`, `COLLECTOR_PROFILE=default|tail`.
Источник истины и транзакционные границы (из lab5, не менялись): PostgreSQL хранит пользователей, чаты, сообщения, метаданные вложений и задания. Upload фиксирует в одной транзакции attachment `queued`, job и строку outbox, и только после COMMIT отвечает 202. Relay публикует outbox в RabbitMQ с publisher confirm. Worker одной транзакцией публикует результат: варианты, `ready`, job `succeeded`. S3 хранит байты, брокер — только ссылки на задания. Telemetry в эти транзакции не входит: trace context пишется в ту же outbox-строку, а spans и метрики экспортируются асинхронно и не могут задержать или отменить COMMIT.
Критичные зависимости `/health/ready` (как в lab5): PostgreSQL и S3. Collector/Tempo/Loki/Prometheus/Grafana **не** критичны — проверено инцидентом C.
## Схема сигналов
Версии: OpenTelemetry SDK 1.30.0, instrumentation 0.51b0, semantic conventions 0.51b0; HTTP-атрибуты по стабильной конвенции (`OTEL_SEMCONV_STABILITY_OPT_IN=http`: `http.request.method`, `http.route`, `http.response.status_code`, метрика `http.server.request.duration` в секундах). Collector — дистрибутив contrib 0.120.0 (нужны `tail_sampling` и `attributes`).
Resource attributes: `service.name` (messenger-api / messenger-worker), `service.namespace=messenger`, `service.version=1.6.0`, `service.instance.id` (api-1/api-2/hostname контейнера worker), `deployment.environment.name=lab`. В Prometheus: `service_name`, `instance`, `service_version`, `deployment_environment_name` (promote_resource_attributes; `job` = `messenger/<service.name>`).
| Сигнал контракта | Инструмент OTel → имя в Prometheus | Labels (все низкокардинальные) | Единица |
| --- | --- | --- | --- |
| HTTP requests (counter) | `http.server.request.duration` → `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`; класс статуса = `label_replace(..., "(\\d)..")` в запросе | запросы |
| HTTP duration | та же гистограмма, `_bucket`; buckets 5 ms … 30 s (View) | те же | s |
| Jobs processed/failed | `messenger.jobs.processed` → `messenger_jobs_processed_total` | `job_type`, `outcome` (succeeded/retry/failed/noop/lost/dlq/requeued) | jobs |
| Job duration | `messenger.job.duration` → `messenger_job_duration_seconds_*`; buckets 50 ms … 120 s | `job_type`, `outcome` | s |
| Queue depth / oldest age | `messenger.queue.depth`, `messenger.queue.oldest_age` → `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | `queue` (attachments.process / outbox), `state` | jobs / s |
| DB pool | `db.client.connection.count`, `.pending_requests`, `.max` → `db_client_connection_*` | `db_client_connection_state` (used/idle) | соединения |
| Worker busy/capacity | `messenger.worker.busy`, `messenger.worker.capacity` | — | jobs |
| Process CPU/RAM | `process.cpu.time` → `process_cpu_time_seconds_total`, `process.memory.usage` → `process_memory_usage_bytes` | `process_cpu_owner` (self/children) | s / bytes |
| End-to-end time to ready | `messenger.attachment.time_to_ready` → `messenger_attachment_time_to_ready_seconds_*`; buckets 0.25 s … 600 s | `job_type` | s |
| Самонаблюдение телеметрии | `messenger.telemetry.sampler.decisions`, `messenger.telemetry.spans.exported`; collector `otelcol_*` | `decision`, `rule`, `outcome`, `exporter` | spans |
| Логи | OTLP log records → Loki | stream labels: `service_name`, `service_instance_id`, `service_namespace`, `deployment_environment_name`; metadata: `event`, `severity_text`, `trace_id`, `span_id`, `code_*` | — |
| Трассы | SERVER (FastAPI), PRODUCER `attachments.process publish`, CONSUMER `attachments.process process`, CLIENT `S3.*` (botocore), CLIENT `INSERT messages` и т.п. (свои), INTERNAL `imageproc`, `db.pool.acquire` | атрибуты: `messenger.request_id`, `messenger.attachment_id`, `messaging.message.id` (= job_id), `messenger.job.attempt/outcome`, `messenger.fault` | — |
Границы buckets подобраны под ожидаемые задержки. HTTP: 5–150 ms — обычные запросы (p50 около 7 ms, p99 около 35–50 ms под нагрузкой); 250 ms – 1 s — граница SLO 500 ms и порог алерта p95 1 s; 2.5–10 s — `lock_timeout` 5 s и `DB_OP_TIMEOUT` 6 s; 30 s — `REQUEST_TIMEOUT`. Job: 50 ms – 3 s — обычная обработка (около 0.15 с, 1 с при ограниченном CPU); 5–120 s — таймаут дочернего процесса 60 s и hard timeout 120 s. Time to ready: 0.25 s – 10 min — от мгновенной обработки до ожидания в очереди с backoff до 30 s и пятью попытками.
Технические ID (`request_id`, `attachment_id`, `job_id`) есть только в span-атрибутах и тексте логов (обоснование: по ним оператор находит трассу конкретной загрузки/ошибки; данные синтетические), никогда в labels метрик — проверено тестом: ни одного UUID в значениях labels, список labels фиксирован. `http.route` — шаблон (`/api/v1/chats/{chat_id}/messages`).
Запрещённое в telemetry: тело сообщения, пароль, Authorization/токен, изображение, filename, signed URL. Защита в два слоя: (1) SDK — заголовки не захватываются, атрибуты HTTP-гистограммы ограничены View, SQL параметризован (`db.query.text` содержит только `%s`), логи не печатают тело/пароль; (2) collector `attributes/scrub` удаляет `url.query` (там был бы `?filename=`), `url.full`, `http.target`, `http.url`, заголовки authorization/cookie. Проверка: `test_observability.py` ищет пароль, токен, текст-маркер, filename, `Authorization`, `Bearer`, `filename=`, `X-Amz-Signature` в трассах и логах — не найдено.
### Перенос контекста API → broker → worker (уровень 4)
1. Upload: server span `POST /api/v1/chats/{chat_id}/attachments`. В той же транзакции, что attachment `queued` и job, в `outbox.payload` записываются `traceparent`/`tracestate` текущего span (поля разрешены `contracts/jobs.schema.json`).
2. Relay outbox (фоновый цикл API) создаёт PRODUCER span **дочерним** контексту из payload и публикует сообщение уже со своим `traceparent` (в теле и дублем в AMQP headers).
3. Worker извлекает контекст из тела (fallback — headers) и создаёт CONSUMER span — дочерний producer. Внутри: `S3.GetObject`, `imageproc`, `S3.CopyObject`, `S3.PutObject`×3, SQL, `S3.DeleteObject`.
4. Retry: `fail_attempt` пишет новую outbox-строку внутри consumer span → следующий PRODUCER — дочерний упавшему CONSUMER. Вся история upload, включая повторы, — **одна трасса**.
5. Sampler `ParentBased(root=KindFilter(TraceIdRatio))`: корнем трассы может быть только SERVER/CONSUMER/PRODUCER, поэтому одиночные SQL-spans фоновых циклов (outbox polling, sweeper, readiness) не создают тысячи мусорных трасс; дети следуют решению родителя, поэтому upload и его job сэмплируются вместе.
Ограничение: восстановление job по истёкшему lease (sweeper) начинает новую трассу (нет активного контекста); связь видна по `messenger.attachment_id`.
Пример (вывод `test_observability.py`, `evidence/observability.txt`; upload с `fault-transient1` — первая попытка падает временно):
```
api SERVER POST /api/v1/chats/{chat_id}/attachments 9.0 ms
api CLIENT S3.PutObject 3.4 ms
api PRODUCER attachments.process publish 1.7 ms attempt=1
worker CONSUMER attachments.process process 6.1 ms ERROR outcome=retry
api PRODUCER attachments.process publish 2.9 ms attempt=2
worker CONSUMER attachments.process process 168.6 ms outcome=succeeded
worker CLIENT S3.GetObject 3.7 ms
worker INTERNAL imageproc 139.6 ms
worker CLIENT S3.CopyObject 4.4 ms
worker CLIENT S3.PutObject (x3) 2.0-2.4 ms
worker CLIENT S3.DeleteObject 1.0 ms
trace 40391c8b40fc76e7f27913c3d84421a4, 42 spans (SQL-spans в выводе опущены). В последнем consumer результат публикуется
SQL-spans: UPDATE attachment_jobs, INSERT attachment_variants ×3, UPDATE attachments, INSERT job_audit. В Loki по этому trace_id 6 записей:
api: attachment_queued; worker: job_claimed, job_attempt_failed, job_finished, job_claimed, job_finished
```
### SLI / SLO и алерты (уровень 4)
- **SLI доступности** = 1 − (5xx / все бизнес-запросы messenger-api); health не инструментирован, 4xx — исход клиента, не отказ. **SLO 99%** на скользящем окне 30 мин (учебное окно; в production те же формулы на 28–30 днях). Recording rule `messenger:slo_availability:ratio30m`.
- **SLI латентности** = доля запросов ≤ 500 ms (bucket `le="0.5"`), **SLO 95%** / 30 мин, `messenger:slo_latency:ratio30m`.
- Перцентили на dashboard — `histogram_quantile` по `sum by (le)` buckets **всех** реплик, а не среднее и не усреднение p99 реплик.
- SLO проверяется значением recording rule в Prometheus. В инциденте B: перед блокировкой доступность 1.0000 и латентность 0.9825; в момент firing 0.6894 и 0.6933; через 2 минуты после снятия блокировки 0.7697 и 0.7616. Оба SLO нарушены, бюджет ошибок окна (1%) исчерпан. Значения: `messenger:slo_availability:ratio30m` и `messenger:slo_latency:ratio30m` на моменты 14:34:58, 14:36:53 и 14:39:09 UTC 7 ноября.
Ключевые панели dashboard (полный перечень 32 панелей с запросами и единицами — `docs/dashboard.md`):
| Панель | Запрос | Единица |
| --- | --- | --- |
| Латентность API p50/p95/p99 | `histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval])))` (и 0.5, 0.95) | секунды |
| Доля ошибок 5xx | `(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]))` | доля |
| RPS по классу статуса | `sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{...}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d).."))` | запросы/с |
| Jobs по исходу | `sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))` | jobs/с |
| Длительность job p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval])))` | секунды |
| Глубина очереди | `max by (queue, state) (messenger_queue_depth)` | jobs |
| Возраст старейшей job | `max(messenger_queue_oldest_age_seconds{queue="attachments.process"})` | секунды |
| Время до ready p50/p95/p99 | `histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval])))` | секунды |
| Пул БД | `sum by (service_name, db_client_connection_state) (db_client_connection_count)`, `sum by (service_name) (db_client_connection_pending_requests)` | соединения, запросы |
| Насыщение worker | `messenger:worker_saturation:ratio` = busy / capacity по свежим сериям | доля |
| CPU / RSS процессов | `sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))`, `process_memory_usage_bytes` | ядра, байты |
| Alert | Условие / окно / порог / for | Проверен |
| --- | --- | --- |
| MessengerApiHighErrorRate | 5xx/все за 2m > 5% при > 0.2 rps, for 1m | инцидент B: firing и resolved |
| MessengerQueueLagGrowing | max(oldest job age) > 60 s, for 1m | инцидент A: firing и resolved |
| MessengerApiLatencyHigh | p95 (5m) > 1 s, for 2m | инцидент B: firing 14:38:00–14:41:45 UTC, затем resolved (`evidence/alerts_history.txt`) |
| MessengerDbPoolWaiting | waiting > 0, for 1m | инцидент B: firing 14:36:30–14:37:00 UTC, затем resolved (`evidence/alerts_history.txt`) |
| MessengerWorkerSaturated | busy/capacity > 0.9 (только свежие серии), for 5m | `test_worker_saturation.py`: firing и resolved |
| TelemetryExportFailing | заполнение очереди exporter > 20% или отбросы/исчерпанные ретраи, for 1m | инцидент C1: firing и resolved |
Runbooks: `docs/runbooks.md`. Состояние — Prometheus `/api/v1/alerts` и панель «Алерты Prometheus» на dashboard.
### Collector: память и недоступный backend
`memory_limiter` первым в каждом pipeline (limit 180 MiB, spike 40 MiB при лимите контейнера 256 MiB): при превышении receiver отвечает отказом, SDK ретраит/отбрасывает. Каждый exporter: `sending_queue` в памяти на 100 батчей (≤ 2048 элементов) и `retry_on_failure` с `max_elapsed_time: 120s` — ни бесконечного буфера, ни бесконечных повторов. На стороне SDK: BatchSpanProcessor/BatchLogRecordProcessor с очередью 2048, при переполнении — отброс; экспорт идёт в отдельном потоке, запросы API не ждут telemetry.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke при включённой телеметрии | `make check`, `make validate`, `make test` | все PASS | `evidence/smoke.txt` |
| Воспроизводимость из чистого clone | копия только файлов git, шаги job `.github/workflows/stand.yml`, отдельный compose-проект | все сервисы healthy, smoke PASS, dashboard импортирован, 13 правил, метрики идут | `evidence/clean_clone.txt` |
| Минимум lab5 сохранён | `test_worker_down.py`, `test_failure_windows.py` (A–I) | PASS | `evidence/lab5_worker_down.txt`, `evidence/lab5_failure_windows.txt` |
| Сохранность данных, реальный рестарт | `make test-persistence` | PASS; uptime всех хранилищ, включая Prometheus, Loki, Tempo, сброшен | `evidence/persistence.txt` |
| Failover каждой реплики | `make test-failover` ×2 | PASS; 0 неудачных попыток, восстановление 2.0 и 0.9 с | `evidence/failover.txt` |
| 3: Collector, 3 хранилища, Grafana в общем стенде; конфиги в Git | `compose.yaml`, `infra/` | стенд поднимается одной командой | `evidence/clean_clone.txt` |
| 3: volumes сохраняют историю | рестарт Prometheus, Loki, Tempo, Grafana | трасса, логи и алерты 7 ноября доступны 9 ноября после рестарта | `evidence/telemetry_persistence.txt` |
| 3: RPS/errors/duration, jobs, duration worker в Grafana | `check_dashboard.py` | 32 панели; все 44 запроса Prometheus, запросы Loki и Tempo выполнены через саму Grafana с данными; Grafana видит 6 правил алертов | `evidence/dashboard_check.txt` |
| 3: контролируемый запрос → trace → лог; job → worker trace → лог | `test_observability.py` | PASS | `evidence/observability.txt` |
| 3: нет секретов, изображений и ID в метках | `test_observability.py` | пароль, токен, текст, filename, Authorization, signed URL и байты PNG (в сыром виде и base64) не найдены; UUID в labels нет | `evidence/observability.txt` |
| 4: W3C API → broker → worker, цепочка до S3/DB с retry | `test_observability.py` | одна трасса, 42 spans, consumer — дочерний producer, retry — дочерний упавшего consumer | `evidence/observability.txt` |
| 4: p50/p95/p99, ошибки, очередь, DB pool, saturation | dashboard и `check_dashboard.py` | панели с запросами и единицами; перцентили из buckets | `infra/grafana/dashboards/messenger.json` |
| 4: SLI/SLO и alert rules проверены отказом | `test_incidents.py`, `test_worker_saturation.py` | все 6 алертов: firing → resolved | `evidence/incidents.txt`, `evidence/alerts_history.txt`, `evidence/worker_saturation.txt` |
| 4: memory limiter, batch, ограниченный буфер | Tempo остановлен при 100 RPS | очередь 100/100, отброшено 25147 и отказано 16802 spans, RSS collector до 206 MiB при лимите 256 MiB, API без ошибок | `evidence/exporter_queue_full.txt` |
| 4: остановка telemetry не ломает сообщения | инцидент C | 0 неожиданных ошибок API при остановке backend и collector | `evidence/incidents.txt` |
| 4: разбор двух инцидентов | разделы ниже | шаги диагностики и причины | `evidence/incidents.txt`, `docs/runbooks.md` |
| 5, трек 1 | `track1.py run --repeats 3` | 4 конфигурации, по 3 прогона | `evidence/track1/results.jsonl`, `evidence/track1/summary.md`, `evidence/track1/matrix.log` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Перезапуск в `test-persistence` затрагивает API, worker, LB, PostgreSQL, Valkey, MinIO, RabbitMQ и, с lab6, Prometheus, Loki, Tempo, collector и Grafana (`scripts/restart.sh`), подтверждено состояниями контейнеров до и после.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Стенд: macOS arm64, Docker 29.7.2, VM 10 CPU / 7.7 GiB. Лимиты: API 0.5 CPU / 512 MiB ×2, worker 2 CPU / 1.5 GiB ×2, PostgreSQL 1 CPU / 512 MiB, collector 1 CPU / 256 MiB, Tempo 1 CPU / 1 GiB. Фоновая нагрузка в инцидентах: open-loop 10 RPS (75% POST message, 25% history, 2% `X-Fault: slow`). Времена отсчитываются от внесения отказа; полный вывод в `evidence/incidents.txt`, сводка в `evidence/incidents.json`.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| A. Остановка worker | рост возраста задания, `MessengerQueueLagGrowing` | pending +71 с, firing +131 с при возрасте 126 с; глубина 3 | backlog разобран за 3.6 с, resolved через 14 с; потерь заданий нет |
| B. `LOCK TABLE messages` на 106 с | 5xx, ожидание в БД видно в трассе | pending +45 с, firing +105 с; 5xx 70.7%, p95 6.92 с, пул 10/10, ждут 19 | resolved через 135 с после снятия блокировки (окно 2m); подтверждённые данные не теряются |
| C1. Tempo и Loki остановлены на 192 с | API работает, очередь ограничена | 1703 запроса, 0 неожиданных ошибок; очередь Tempo 79/100, `TelemetryExportFailing` firing | экспорт возобновился, resolved через 35 с; потеряно 387 записей telemetry (ретраи исчерпаны) |
| C2. Collector остановлен на 60 с | API работает, очереди SDK ограничены | 740 запросов, 0 неожиданных ошибок; RSS api-1 108.6 → 109–119 MiB | telemetry за время простоя теряется сверх очереди SDK (2048) |
| Tempo остановлен при 100 RPS, 223 с | очередь упирается в предел, отбросы считаются | очередь 100/100, отброшено 25147, отказано memory_limiter 16802 spans; RSS до 206 MiB | API: 24686 запросов, 0 ошибок, p99 55.8 ms; потерянные spans не восстанавливаются. После возврата Tempo сброс очереди превысил его лимит 15 MB/с: ещё 32410 spans отклонены (`RATE_LIMITED`) и посчитаны в `send_failed` |
| Насыщение worker (1 слот ×2, 0.25 CPU), 4 upload/с | saturation 1.0 дольше `for: 5m` | firing +340 с; saturation 1.00, 194 в очереди, возраст 63 с, 3.16 job/с, p95 job 0.98 с | после возврата настроек resolved через 17 с, backlog разобран за 22 с |
### Инцидент 1: задания не обрабатываются
1. Сработал `MessengerQueueLagGrowing`. Метрики: возраст старейшего задания 126 с, 3 задания в `queued`, `rate(messenger_jobs_processed_total[1m]) = 0`. Это не медленная обработка, задания не берутся вообще.
2. Логи worker: после остановки 0 событий `job_*`, последние события `worker_stopped`.
3. Трасса upload: есть SERVER и PRODUCER, нет CONSUMER. Сообщение дошло до брокера, потребителя нет.
4. Причина: остановлен worker. Действие: `docker compose start worker`. Задания сохранены в PostgreSQL и durable-очереди, поэтому все три стали ready за 3.6 с.
### Инцидент 2: медленная БД или медленный handler
1. Сработал `MessengerApiHighErrorRate`: 5xx 70.7%, p95 6.92 с. Ошибки 503 приходятся на `/api/v1/chats/{chat_id}/messages`.
2. Метрики пула: занято 10 из 10, ожидают 19, поэтому подозрение падает на БД.
3. Трасса упавшего запроса: `INSERT messages` занял 5001 ms из 5004 ms и завершился `QueryCanceled` (срабатывает `lock_timeout`). Время потрачено в PostgreSQL.
4. Контраст: запрос с `X-Fault: slow` длится 615 ms, а его самый долгий DB-span — 1.1 ms, все DB-spans вместе — 8.3 ms. Так выглядит медленный handler.
5. Шаг оператора: `pg_stat_activity` показывает блокирующий `LOCK TABLE messages`; после `pg_terminate_backend` ошибки прекращаются.
Побочный эффект, найденный в этом инциденте: исчерпание пула делает `/health/ready` неуспешным (событие `not_ready`), поэтому блокировка одной таблицы отключает обе реплики целиком. За время инцидента клиент получил 816 ответов 503 и 214 ответов 502 от nginx.
## Трек 5.1: цена наблюдаемости
**Гипотезы.** H1: стоимость телеметрии в приложении пропорциональна числу записанных spans, поэтому head sampling снижает CPU, а tail sampling — нет. H2: head sampling 10% теряет около 90% ошибок и редких медленных запросов, потому что решение принимается до их исхода. H3: tail sampling сохраняет все такие трассы и резко сокращает объём в хранилище. H4: метрики от сэмплирования не зависят.
**Методика.** Конфигурации: `off` (`OTEL_SDK_DISABLED=true`), `on100` (100%), `head10` (head sampling 10% корневых spans), `tail` (SDK 100%; collector оставляет ошибки, трассы > 500 ms, конвейер вложений и 10% остального; `decision_wait` 10 с). Перед каждым прогоном пересоздаются API, worker и collector. Профиль: open-loop 60 RPS (75% POST message, 25% GET history), 1 upload/с, 1% запросов с `+600 ms`, 1% запросов с ответом 500; прогрев 20 с, пауза 20 с, замер 90 с, пауза 30 с. Три прогона каждой конфигурации, порядок циклический. Объём данных: перед экспериментом в БД 16 739 сообщений, 38 чатов, 89 пользователей, 87 вложений; после — 75 179, 130, 273 и 1 277 (каждый прогон создаёт 4 новых чата, история самого большого чата 2 701 сообщение). GET history читает первую страницу (limit=20), поэтому рост БД на latency чтения почти не влияет. Latency считается от запланированного момента отправки (без coordinated omission), только для успешных обычных запросов. CPU — точная разность cgroup-счётчика за окно; для collector сверена с его собственной метрикой (0.52 и 0.54% ядра). Запуск: `evidence/track1/matrix.log`.
Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.
| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |
|---|---|---|---|---|---|---|---|---|
| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) |
| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) |
| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) |
| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) |
| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |
|---|---|---|---|---|---|---|---|
| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 |
| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 |
| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 |
| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 |
| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |
|---|---|---|---|---|---|---|
| off | — | 0 | 0 | — | 0 | 0 |
| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 |
| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 |
| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 |
Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.
| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |
|---|---|---|---|---|---|
| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 |
| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 |
| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 |
| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 |
**Результаты по гипотезам.**
- **H1 подтверждена.** CPU API относительно `off`: `on100` +4.9 п.п. (+28%, около 0.8 ms CPU на запрос), `head10` +1.9 п.п. (+11%), `tail` +4.5 п.п. (+25%). Диапазоны между прогонами не пересекаются. CPU worker не изменился (15.0–15.6%): его определяет обработка изображений.
- **Latency.** p99: `off` 32.7, `head10` 37.5, `tail` 44.5, `on100` 47.2 ms; диапазоны `off` и `on100` не пересекаются. p50 различается меньше, чем прогоны одной конфигурации (6.4–8.7 ms), поэтому разницы по p50 заявить нельзя. p95 у `on100` и `tail` выше, но диапазоны частично пересекаются с `off`.
- **H2 подтверждена.** При `head10` сохранилось 18/198 медленных запросов (9%), 15/166 ошибок (9%) и 25/270 конвейеров вложений (9%).
- **H3 подтверждена.** `tail` сохранил 180/180, 164/164 и 270/270 при сокращении ingest Tempo на 81% (20547 → 3877 KiB) и числа spans на 83%. Обычных трасс осталось около 14%. Экономии между SDK и collector нет: 21 MiB за окно, как в `on100`.
- **H4 подтверждена.** Счётчик HTTP 500 совпал с числом отправленных ошибок во всех режимах: 146/146, 166/166, 164/164. При `head10` метрика видит все 166 ошибок, а разобрать по трассе можно только 15.
- **Потери при передаче** отсутствуют: `Spans → collector` равно `Spans → Tempo` в `on100` и `head10`, отказов и отбросов collector нет, SDK не сообщил об ошибках экспорта.
**Что теряется.** `head10`: около 90% любых трасс, включая ошибки и редкие медленные запросы. `tail`: 90% обычных успешных запросов; запросы медленнее обычного, но быстрее порога 500 ms; ответы 4xx (их server span не имеет статуса ERROR). Части трассы, пришедшие позже `decision_wait` (10 с) после решения об отбросе, тоже будут потеряны; для конвейера вложений это закрыто политикой по `messaging.system`.
**Сбалансированная конфигурация: `tail`** (`COLLECTOR_PROFILE=tail`). Её цена: те же +25% CPU API, что у полной трассировки, и +12 ms к p99, зато сохраняются все диагностически важные трассы при объёме хранения на 81% меньше. `head10` разумен, только если критичен CPU приложения и достаточно метрик и логов. По умолчанию в репозитории оставлен `on100`: на защите нужно найти любую контрольную трассу, а нагрузка учебная.
**Ограничения измерения.** Один Docker-host и генератор на той же машине, API далёк от насыщения (около 11% ядра на реплику), поэтому поведение под перегрузкой здесь не измерено. Три прогона по 90 с дают ориентировочные оценки. RSS Tempo между конфигурациями не сравним: контейнер не пересоздавался. Колонка SDK-решений из сводки исключена: счётчики пересозданных контейнеров выпадали из окна запроса и давали неверные разности. Tail sampling требует, чтобы все spans трассы приходили в один collector; при нескольких collector нужна маршрутизация по trace_id.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Выдерживается: отказ одной реплики API, остановка и перезапуск worker и брокера с volumes, недоступность Tempo, Loki или Collector без влияния на бизнес-API (данные telemetry при этом теряются сверх ограниченных очередей, потери считаются).
Не выдерживается и не заявляется: потеря Docker-host, отказ единственных PostgreSQL, nginx или collector как высокодоступных компонентов. Хранилища telemetry работают в одном экземпляре.
Известные ограничения и упрощения:
- Блокировка БД исчерпывает пул и переводит реплики в not ready, поэтому отказ одной таблицы останавливает весь API. Улучшение: отдельное соединение для readiness. Пока это гипотеза, не реализовано.
- OTLP-метрики в Prometheus не получают staleness-маркеров: серии остановленного процесса видны ещё 5 минут. Правило насыщения worker это учитывает; в остальных запросах значения исчезают с задержкой.
- `MessengerApiHighErrorRate` срабатывает и на 503 от admission control очереди (видно в тесте насыщения): это реальный отказ клиенту, но причину надо смотреть по маршруту и логам.
- Восстановление задания по истёкшему lease начинает новую трассу; связь видна по `messenger.attachment_id`.
- Метрики CPU worker учитывают дочерние процессы только после их завершения; RSS дочерних процессов не учитывается.
- Первый прогон инцидентов сорван сном Mac: стенд замер на 18 минут. Длинные прогоны нужно запускать под `caffeinate -i`.
- `FAULT_INJECTION=1` и заголовок `X-Fault` допустимы только на учебном стенде. HTTP без TLS — только локально, TLS и mTLS относятся к lab7.
- После долгого простоя Tempo сброс накопленной очереди упирается в лимит ingest: часть данных теряется уже после восстановления, алерт остаётся firing ещё около 2 минут (окно rate).
- Окно SLO 30 минут учебное.
Следующие шаги: отдельный пул для readiness, TLS в lab7, проверка tail sampling под нагрузкой, близкой к насыщению.
View File
+79
View File
@@ -0,0 +1,79 @@
"""RabbitMQ topology and confirmed publishing (aio-pika).
Topology (declared idempotently by every publisher and consumer):
exchange attachments direct, durable
queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit
exchange attachments.dlx fanout, durable
queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator)
Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker
has taken responsibility for the message (written to the quorum log).
"""
import json
import logging
import aio_pika
from aio_pika import DeliveryMode, ExchangeType, Message
from app import config
log = logging.getLogger("messenger.broker")
ROUTING_KEY = "attachment.process.v1"
async def connect():
return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, client_properties={"connection_name": config.INSTANCE_ID})
async def declare(channel):
dlx = await channel.declare_exchange(config.BROKER_DLX, ExchangeType.FANOUT, durable=True)
dlq = await channel.declare_queue(config.BROKER_DLQ, durable=True, arguments={"x-queue-type": "quorum"})
await dlq.bind(dlx)
exchange = await channel.declare_exchange(config.BROKER_EXCHANGE, ExchangeType.DIRECT, durable=True)
queue = await channel.declare_queue(
config.BROKER_QUEUE,
durable=True,
arguments={
"x-queue-type": "quorum",
"x-dead-letter-exchange": config.BROKER_DLX,
"x-delivery-limit": config.BROKER_DELIVERY_LIMIT,
},
)
await queue.bind(exchange, ROUTING_KEY)
return exchange, queue
class Publisher:
def __init__(self) -> None:
self.connection = None
self.channel = None
self.exchange = None
async def ensure(self) -> None:
if self.channel is not None and not self.channel.is_closed:
return
if self.connection is None or self.connection.is_closed:
self.connection = await connect()
self.channel = await self.connection.channel(publisher_confirms=True)
self.exchange, _ = await declare(self.channel)
async def publish(self, body: dict) -> None:
await self.ensure()
message = Message(
json.dumps(body, separators=(",", ":")).encode(),
content_type="application/json",
delivery_mode=DeliveryMode.PERSISTENT,
message_id=body["job_id"],
type=body["type"],
# Native AMQP headers duplicate the body carrier (same W3C values) for broker-side tooling.
headers={"attempt": body["attempt"], **{k: body[k] for k in ("traceparent", "tracestate") if k in body}},
)
# mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack.
await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5)
async def close(self) -> None:
try:
if self.connection is not None:
await self.connection.close()
except Exception:
pass
self.connection = self.channel = self.exchange = None
+98
View File
@@ -0,0 +1,98 @@
import os
import socket
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
path = os.environ.get(name + "_FILE")
if path:
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
return os.environ.get(name, default)
DATABASE_URL = _secret("DATABASE_URL")
CACHE_URL = _secret("CACHE_URL")
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
READY_FAILURES = _int("READY_FAILURES", 2)
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
PASSWORD_MIN = 12
PASSWORD_MAX = 128
SESSION_REVOKED_TTL_SECONDS = max(
_int("SESSION_REVOKED_TTL_SECONDS", 60),
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
)
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
# --- lab5: broker, jobs, worker ---
BROKER_URL = _secret("BROKER_URL")
BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments")
BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx")
BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq")
BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20)
PIPELINE_VERSION = _int("PIPELINE_VERSION", 1)
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2)
WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY
JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5)
JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60)
JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20)
JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0)
JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0)
JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0)
WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768)
WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25)
WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat")
OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0)
OUTBOX_BATCH = _int("OUTBOX_BATCH", 50)
OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400)
STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400)
# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables).
MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200)
PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0)
THUMBNAIL_SIZE = 256
CROP_SIZE = 128
OPTIMIZED_SIZE = 1280
JPEG_QUALITY = _int("JPEG_QUALITY", 85)
WEBP_QUALITY = _int("WEBP_QUALITY", 80)
# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py).
FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1"
+43
View File
@@ -0,0 +1,43 @@
import base64
import json
import re
import uuid
from datetime import datetime, timezone
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
class BadCursor(Exception):
pass
def fmt(ts: datetime) -> str:
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
def decode_cursor(scope: str, cursor: str):
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
data = json.loads(raw)
except Exception:
raise BadCursor() from None
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
raise BadCursor()
ts, item_id = data["t"], data["i"]
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
raise BadCursor()
try:
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
if str(uuid.UUID(item_id)) != item_id:
raise BadCursor()
except ValueError:
raise BadCursor() from None
if encode_cursor(scope, ts, item_id) != cursor:
raise BadCursor()
return parsed, item_id
+134
View File
@@ -0,0 +1,134 @@
import asyncio
import re
import time
from contextlib import asynccontextmanager
import psycopg
from opentelemetry.trace import SpanKind, Status, StatusCode
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from app import config, telemetry
from app.errors import unavailable
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
OPERATION_RE = re.compile(r"^\s*([A-Za-z]+)")
TABLE_RE = re.compile(r"\b(?:FROM|INTO|UPDATE|JOIN)\s+([a-z_]+)", re.IGNORECASE)
POOL_WAIT_SPAN_SECONDS = 0.005
def statement(query) -> tuple:
"""Span name and attributes for a parametrised SQL statement (values are never part of the text)."""
text = query if isinstance(query, str) else str(query)
op = OPERATION_RE.match(text)
table = TABLE_RE.search(text)
operation = op.group(1).upper() if op else "SQL"
name = operation + (" " + table.group(1) if table else "")
return name, {"db.system": "postgresql", "db.operation.name": operation, "db.namespace": "messenger",
"db.collection.name": table.group(1) if table else "", "db.query.text": text[:400]}
async def discard(conn) -> None:
try:
await conn.close()
except Exception:
pass
class Guarded:
def __init__(self, conn, timeout: float):
self.conn = conn
self.timeout = timeout
async def execute(self, *args, **kwargs):
# Own CLIENT span around the awaited statement: the time really spent in PostgreSQL (lock waits included).
# (opentelemetry-instrumentation-psycopg 0.51b0 ends async spans before the await completes -> 0 ms spans.)
name, attrs = statement(args[0] if args else "")
with telemetry.span(name, kind=SpanKind.CLIENT, attributes=attrs) as span:
try:
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
except asyncio.TimeoutError:
await discard(self.conn)
span.set_status(Status(StatusCode.ERROR, "client timeout %.1fs" % self.timeout))
raise unavailable("Хранилище не ответило за отведённое время") from None
def transaction(self):
return self.conn.transaction()
class Database:
def __init__(self) -> None:
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS * 2,
)
self.pool = AsyncConnectionPool(
conninfo=config.DATABASE_URL,
min_size=config.DB_POOL_MIN,
max_size=config.DB_POOL_MAX,
timeout=config.DB_POOL_TIMEOUT,
max_lifetime=600.0,
open=False,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": config.DB_CONNECT_TIMEOUT,
"options": options,
},
)
async def open(self) -> None:
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close()
@asynccontextmanager
async def connection(self):
started, acquired = time.time_ns(), False
try:
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
acquired = True
waited = (time.time_ns() - started) / 1e9
if waited >= POOL_WAIT_SPAN_SECONDS: # only noticeable waits become spans
telemetry.tracer.start_span("db.pool.acquire", start_time=started,
attributes={"db.system": "postgresql",
"db.client.connection.wait_s": round(waited, 4)}).end()
yield Guarded(conn, config.DB_OP_TIMEOUT)
except BROKEN as exc:
if not acquired:
span = telemetry.tracer.start_span("db.pool.acquire", start_time=started,
attributes={"db.system": "postgresql"})
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
span.end()
raise unavailable("Хранилище недоступно") from None
@asynccontextmanager
async def transaction(self):
async with self.connection() as conn:
async with conn.transaction():
yield conn
async def in_transaction(self, action):
async def run():
async with self.transaction() as conn:
return await action(conn)
try:
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
except asyncio.TimeoutError:
raise unavailable("Хранилище не ответило за отведённое время") from None
async def ready(self) -> bool:
try:
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
try:
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
except asyncio.TimeoutError:
await discard(conn)
return False
return True
except Exception:
return False
+37
View File
@@ -0,0 +1,37 @@
from typing import Optional
from fastapi.responses import JSONResponse
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(scheme: str = "Bearer") -> ApiError:
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found() -> ApiError:
return ApiError(404, "not_found", "Объект не найден")
def unavailable(message: str) -> ApiError:
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": rid}},
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
)
+38
View File
@@ -0,0 +1,38 @@
"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off).
A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`.
Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata.
Crash faults fire only on the first attempt and first delivery, so recovery can be observed.
api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish
worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT
worker_crash_after_commit worker exits after the result COMMIT, before broker ack
worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1
transientN attempts 1..N fail with a transient error (N = 1..9)
permanent permanent error on the first attempt -> failed without retries
"""
import logging
import os
import re
from typing import Optional
from app import config
log = logging.getLogger("messenger.faults")
TRANSIENT_RE = re.compile(r"fault-transient([1-9])")
def active(filename: Optional[str], name: str) -> bool:
return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename
def transient_until(filename: Optional[str]) -> int:
if not config.FAULT_INJECTION or not filename:
return 0
match = TRANSIENT_RE.search(filename)
return int(match.group(1)) if match else 0
def crash(point: str) -> None:
log.error("FAULT INJECTION: process exits at %s", point)
logging.shutdown()
os._exit(70)
+213
View File
@@ -0,0 +1,213 @@
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
only a warning. Both finish before anything is stored.
"""
import logging
import os
import struct
import subprocess
import threading
import time
import zlib
from PIL import Image
from app import config
from app.errors import ApiError, unavailable
log = logging.getLogger("messenger.imagecheck")
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
Image.MAX_IMAGE_PIXELS = None
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
FULL_DECODE_MAX_PIXELS = 4000000
SCAN_SECONDS = 10.0
OUT_STEP = 1 << 20
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
jpeg_slot = threading.Lock()
class Invalid(Exception):
pass
def invalid() -> ApiError:
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
class RowPlan:
def __init__(self, segments):
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
self.seg = 0
self.done = 0
self.pos = 0
def feed(self, data: bytes) -> None:
i, n = 0, len(data)
while i < n:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
if self.seg >= len(self.segments):
raise Invalid("data after the last row")
length, rows = self.segments[self.seg]
if self.pos > 0:
take = min(length - self.pos, n - i)
i += take
self.pos += take
if self.pos == length:
self.pos = 0
self.done += 1
continue
full = min((n - i) // length, rows - self.done)
if full > 0:
if max(data[i:i + full * length:length]) > 4:
raise Invalid("bad filter type")
i += full * length
self.done += full
else:
if data[i] > 4:
raise Invalid("bad filter type")
self.pos = n - i
i = n
def complete(self) -> bool:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
return self.seg >= len(self.segments) and self.pos == 0
def png_plan(width, height, depth, color, interlace):
bits = CHANNELS[color] * depth
if not interlace:
return RowPlan([(1 + (width * bits + 7) // 8, height)])
segments = []
for x0, y0, dx, dy in ADAM7:
w = (width - x0 + dx - 1) // dx if width > x0 else 0
h = (height - y0 + dy - 1) // dy if height > y0 else 0
if w and h:
segments.append((1 + (w * bits + 7) // 8, h))
return RowPlan(segments)
def scan_png(fileobj) -> None:
deadline = time.monotonic() + SCAN_SECONDS
fileobj.seek(0)
if fileobj.read(8) != PNG_SIGNATURE:
raise Invalid("signature")
plan = decomp = None
palette = seen_idat = idat_closed = False
color = None
first = True
while True:
if time.monotonic() > deadline:
raise Invalid("scan too slow")
head = fileobj.read(8)
if len(head) < 8:
raise Invalid("truncated before IEND")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise Invalid("chunk length")
if first and kind != b"IHDR":
raise Invalid("IHDR must be first")
crc = zlib.crc32(kind)
if kind == b"IHDR":
body = fileobj.read(length)
if not first or length != 13 or len(body) != 13:
raise Invalid("IHDR")
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
raise Invalid("IHDR fields")
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
raise Invalid("color type / depth")
plan = png_plan(width, height, depth, color, interlace)
decomp = zlib.decompressobj()
crc = zlib.crc32(body, crc)
else:
remaining = length
while remaining:
buf = fileobj.read(min(remaining, 65536))
if not buf:
raise Invalid("truncated chunk")
remaining -= len(buf)
crc = zlib.crc32(buf, crc)
if kind == b"IDAT":
if idat_closed or (color == 3 and not palette):
raise Invalid("IDAT order")
seen_idat = True
out = decomp.decompress(buf, OUT_STEP)
while True:
plan.feed(out)
if not decomp.unconsumed_tail:
break
if time.monotonic() > deadline:
raise Invalid("scan too slow")
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
if kind == b"PLTE":
palette = True
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise Invalid("PLTE")
elif kind != b"IDAT" and seen_idat:
idat_closed = True
tail = fileobj.read(4)
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
raise Invalid("crc")
first = False
if kind == b"IEND":
if length or not seen_idat:
raise Invalid("IEND")
break
if not decomp.eof or decomp.unused_data or not plan.complete():
raise Invalid("pixel data does not match IHDR")
def check_jpeg(fileobj) -> None:
fileobj.flush()
descriptor = fileobj.fileno()
os.lseek(descriptor, 0, os.SEEK_SET)
try:
with jpeg_slot:
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=SCAN_SECONDS, check=False)
except subprocess.TimeoutExpired:
log.error("jpeg_check_timeout")
raise unavailable("Проверка изображения не успела завершиться") from None
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
if done.returncode < 0:
log.error("jpeg_check_killed signal=%d", -done.returncode)
raise unavailable("Проверка изображения прервана")
if done.returncode != 0:
raise invalid()
def inspect_image(fileobj, size: int, content_type: str) -> None:
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
try:
fileobj.seek(0)
img = Image.open(fileobj)
width, height = img.size
if img.format != FORMATS[content_type]:
raise invalid()
if width * height > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
if content_type == "image/png":
scan_png(fileobj)
if width * height <= FULL_DECODE_MAX_PIXELS:
fileobj.seek(0)
Image.open(fileobj).load()
else:
check_jpeg(fileobj)
except ApiError:
raise
except Exception:
raise invalid() from None
finally:
fileobj.seek(0)
+115
View File
@@ -0,0 +1,115 @@
"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a
wall-clock timeout, so a hostile or huge image can only kill this process, never the worker.
usage: python -m app.imageproc <source> <out_dir> <max_pixels>
stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex.
Rules (contracts/README.md "Фоновая обработка"):
thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px
crop JPEG, centred square scaled to 128x128 (small sources are upscaled)
optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4
EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha).
EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB).
"""
import json
import math
import os
import sys
from PIL import Image, ImageFile, ImageOps
Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding
ImageFile.LOAD_TRUNCATED_IMAGES = False
THUMB, CROP, OPTIMIZED = 256, 128, 1280
JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85"))
WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80"))
ORIENTATION_TAG = 0x0112
EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5
def fit(width: int, height: int, bound: int):
"""Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales."""
if width <= bound and height <= bound:
return width, height
if width >= height:
return bound, max(1, height * bound // width)
return max(1, width * bound // height), bound
def has_alpha(img) -> bool:
return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info)
def to_8bit(img):
if img.mode in ("I;16", "I;16B", "I;16L", "I"):
return img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
return img
def on_white(img):
if has_alpha(img):
rgba = img.convert("RGBA")
flat = Image.new("RGB", rgba.size, (255, 255, 255))
flat.paste(rgba, mask=rgba.getchannel("A"))
return flat, rgba
return img.convert("RGB"), None
def process(source: str, out_dir: str, max_pixels: int) -> list:
img = Image.open(source)
if img.format not in ("PNG", "JPEG"):
raise ValueError("format")
width, height = img.size
if width * height > max_pixels:
sys.exit(EXIT_TOO_LARGE)
orientation = img.getexif().get(ORIENTATION_TAG, 1)
upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height)
thumb_size = fit(*upright, THUMB)
opt_size = fit(*upright, OPTIMIZED)
if img.format == "JPEG":
# DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept).
need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0])
img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1])))
img.load()
img = ImageOps.exif_transpose(img)
img = to_8bit(img)
flat, rgba = on_white(img)
del img
results = []
def save(name, image, ext, fmt, content_type, **params):
path = os.path.join(out_dir, name + "." + ext)
image.save(path, fmt, **params)
results.append({"name": name, "file": path, "content_type": content_type,
"width": image.size[0], "height": image.size[1]})
jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False}
save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0),
"jpg", "JPEG", "image/jpeg", **jpeg)
w, h = flat.size
side = min(w, h)
left, top = (w - side) // 2, (h - side) // 2
save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)),
"jpg", "JPEG", "image/jpeg", **jpeg)
source_for_webp = rgba if rgba is not None else flat
optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize(
opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0)
save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4)
return results
def main() -> None:
source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3])
try:
results = process(source, out_dir, max_pixels)
except MemoryError:
sys.exit(EXIT_TOO_COMPLEX)
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
sys.exit(EXIT_INVALID)
json.dump(results, sys.stdout)
if __name__ == "__main__":
main()
+340
View File
@@ -0,0 +1,340 @@
"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker.
State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it
(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes
`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT.
enqueue nothing -> queued (API, same txn as attachment 'queued')
claim queued(due) -> processing, lease (worker; also takes over an expired lease)
heartbeat processing: lease_until += lease (only while owner is still alive)
complete processing -> succeeded + variants (fenced by lease_owner)
fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts)
sweep_leases processing(expired lease) -> queued | failed
replay failed -> queued (explicit operator action, audited)
"""
import json
import random
import uuid
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from app import config, telemetry
from app.cursor import fmt
JOB_TYPE = "attachment.process.v1"
SCHEMA_VERSION = 1
VARIANT_ORDER = ("thumbnail", "crop", "optimized")
REQUIRED_VARIANTS = VARIANT_ORDER
def backoff_seconds(attempt: int) -> float:
"""Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd)."""
ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)))
return ceiling / 2 + random.uniform(0, ceiling / 2)
def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict:
return {
"schema_version": SCHEMA_VERSION,
"job_id": str(job_id),
"type": JOB_TYPE,
"attachment_id": str(attachment_id),
"pipeline_version": pipeline_version,
"attempt": attempt,
"created_at": fmt(created_at),
}
def parse_message(body: bytes) -> Optional[dict]:
"""Strict check against jobs.schema.json; anything else goes to the DLQ untouched."""
try:
data = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
return None
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}:
return None
if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE:
return None
if "traceparent" in data and not (isinstance(data["traceparent"], str)
and telemetry.TRACEPARENT_RE.match(data["traceparent"])):
return None
if "tracestate" in data and not (isinstance(data["tracestate"], str) and len(data["tracestate"]) <= 512):
return None
for key in ("pipeline_version", "attempt"):
if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1:
return None
try:
data["job_id"] = str(uuid.UUID(data["job_id"]))
data["attachment_id"] = str(uuid.UUID(data["attachment_id"]))
except (ValueError, AttributeError, TypeError):
return None
return data
async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None):
await conn.execute(
"INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())",
(job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})),
)
async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0):
body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"])
# W3C trace context of the transaction that schedules the job (upload request span or the failed consumer
# span for a retry) travels in the job message (jobs.schema.json traceparent/tracestate).
body.update(telemetry.inject())
await conn.execute(
"INSERT INTO outbox (job_id, payload, available_at, created_at)"
" VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())",
(job["id"], json.dumps(body), delay),
)
async def enqueue(conn, attachment_id: str, actor: str) -> dict:
cur = await conn.execute(
"INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts,"
" next_attempt_at, created_at, updated_at)"
" VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())"
" RETURNING id, attachment_id, pipeline_version, attempt, created_at",
(attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS),
)
job = await cur.fetchone()
await _outbox(conn, job, 1)
await audit(conn, job["id"], attachment_id, "enqueued", actor, 1)
return job
async def pending_count(conn) -> int:
cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')")
return (await cur.fetchone())["n"]
async def queue_stats(conn) -> dict:
"""Backlog for metrics: queued / processing jobs, age of the oldest unfinished job, unpublished outbox rows."""
cur = await conn.execute(
"SELECT count(*) FILTER (WHERE state = 'queued') AS queued,"
" count(*) FILTER (WHERE state = 'processing') AS processing,"
" coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)::float8 AS oldest_age"
" FROM attachment_jobs WHERE state IN ('queued', 'processing')"
)
row = dict(await cur.fetchone())
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
row["outbox"] = (await cur.fetchone())["n"]
return row
# ---------------------------------------------------------------- worker side
JOB_COLUMNS = (
"j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until,"
" j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired,"
" (j.next_attempt_at <= clock_timestamp()) AS due"
)
async def claim(conn, msg: dict, owner: str):
"""Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version."""
cur = await conn.execute(
"SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status"
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.id = %s FOR UPDATE OF j",
(msg["job_id"],),
)
job = await cur.fetchone()
if job is None or str(job["attachment_id"]) != msg["attachment_id"]:
return "missing", None
if job["state"] in ("succeeded", "failed"):
return "done", job
if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]:
return "newer_version", job
if msg["attempt"] != job["attempt"]:
return "stale", job
if job["state"] == "processing" and not job["lease_expired"]:
return "busy", job
if job["state"] == "queued" and not job["due"]:
return "early", job
takeover = job["state"] == "processing"
await conn.execute(
"UPDATE attachment_jobs SET state = 'processing', lease_owner = %s,"
" lease_until = clock_timestamp() + make_interval(secs => %s),"
" started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp()),"
" updated_at = clock_timestamp() WHERE id = %s",
(owner, config.JOB_LEASE_SECONDS, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(job["attachment_id"],),
)
await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"])
return "claimed", job
async def heartbeat(conn, job_id, owner: str) -> bool:
cur = await conn.execute(
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
(config.JOB_LEASE_SECONDS, job_id, owner),
)
return cur.rowcount == 1
async def attachment_source(conn, attachment_id) -> Optional[dict]:
cur = await conn.execute(
"SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status"
" FROM attachments WHERE id = %s",
(attachment_id,),
)
return await cur.fetchone()
async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool:
"""Atomically publish the result. False when the lease was lost: someone else owns the effect now."""
cur = await conn.execute(
"UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL,"
" lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s",
(job["id"], owner),
)
if cur.rowcount != 1:
return False
for v in variants:
await conn.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,"
" size_bytes, width, height, sha256, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())"
" ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
(job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"],
v["size_bytes"], v["width"], v["height"], v["sha256"]),
)
# Never move a newer pipeline_version back, never touch an attachment that is already ready.
await conn.execute(
"UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()"
" WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s",
(job["pipeline_version"], job["attachment_id"], job["pipeline_version"]),
)
await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"],
{"variants": [v["name"] for v in variants]})
return True
async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str:
attempt = job["attempt"]
if transient and attempt < job["max_attempts"]:
delay = backoff_seconds(attempt)
await conn.execute(
"UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL,"
" next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s,"
" updated_at = clock_timestamp() WHERE id = %s",
(delay, detail[:500], error_code, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(job["attachment_id"],),
)
await _outbox(conn, job, attempt + 1, delay=delay)
await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt,
{"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)})
return "retry"
final_code = error_code if not transient else "retry_exhausted"
await conn.execute(
"UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp(),"
" last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s",
(detail[:500], final_code, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()"
" WHERE id = %s AND status IN ('queued', 'processing')",
(final_code, job["attachment_id"]),
)
await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt,
{"error_code": final_code, "cause": error_code, "detail": detail[:200]})
return "failed"
async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str:
"""Returns retry | failed | lost (lease is not ours any more: the new owner decides)."""
cur = await conn.execute(
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs"
" WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE",
(job["id"], owner),
)
row = await cur.fetchone()
if row is None:
return "lost"
return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient)
async def sweep_leases(conn, actor: str) -> List[str]:
"""Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt."""
cur = await conn.execute(
"SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner"
" FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()"
" ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED",
)
outcomes = []
for row in await cur.fetchall():
outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired",
"lease of %s expired" % (row["lease_owner"] or "?"), True)
outcomes.append("%s:%s" % (row["id"], outcome))
return outcomes
async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict:
"""failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed."""
cur = await conn.execute(
"SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at,"
" a.status AS attachment_status, a.staging_key"
" FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a",
(attachment_id,),
)
job = await cur.fetchone()
if job is None:
return {"result": "refused", "reason": "no job for attachment"}
if job["state"] != "failed" or job["attachment_status"] != "failed":
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"],
{"state": job["state"], "attachment_status": job["attachment_status"]})
return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])}
if not job["staging_key"] or not await staging_exists(job["staging_key"]):
await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"})
return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"}
attempt = job["attempt"] + 1
await conn.execute(
"UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp(),"
" finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s",
(attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]),
)
await conn.execute(
"UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'failed'",
(job["attachment_id"],),
)
await _outbox(conn, job, attempt)
await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt)
return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt}
async def failed_jobs(conn, limit: int = 50) -> List[dict]:
cur = await conn.execute(
"SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at"
" FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s",
(limit,),
)
return [
{"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"],
"max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"],
"failed_at": fmt(r["updated_at"])}
for r in await cur.fetchall()
]
def utcnow() -> datetime:
return datetime.now(timezone.utc)
def older_than(seconds: int) -> datetime:
return utcnow() - timedelta(seconds=seconds)
+721
View File
@@ -0,0 +1,721 @@
import asyncio
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import resource
import signal
import tempfile
import unicodedata
import sys
import uuid
from contextlib import asynccontextmanager
from typing import List, Optional, Tuple
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
from starlette.responses import StreamingResponse
from app import config, faults, jobs, passwords, reconcile, repo, telemetry
from app.cursor import BadCursor, decode_cursor, encode_cursor
from app.db import Database
from app.imagecheck import inspect_image
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
from app.broker import Publisher
from app.outbox import Relay
from app.s3 import S3Storage
from app.sessions import SessionStore
log = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"})
EXEMPT_PATHS = ("/health/live", "/health/ready")
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
JPEG_MAGIC = b"\xff\xd8\xff"
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
background = set()
db = Database()
sessions = SessionStore(db)
s3 = S3Storage()
publisher = Publisher()
relay = Relay(db, publisher)
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0,
"queue": None}
VARIANTS = ("original", "thumbnail", "crop", "optimized")
def rss_mb() -> float:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
except OSError:
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
def count_rejection(code: str) -> None:
state["rejected"][code] = state["rejected"].get(code, 0) + 1
async def sweeper() -> None:
while True:
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
try:
removed = await sessions.sweep()
if removed:
log.info("sessions_expired removed=%d", removed)
except Exception:
log.warning("session_sweep_skipped: storage unavailable")
try:
async with db.connection() as conn:
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
if removed:
log.info("idempotency_cleanup removed=%d", removed)
except Exception:
log.warning("idempotency_cleanup_skipped")
try:
removed = await relay.cleanup()
if removed:
log.info("outbox_cleanup removed=%d", removed)
except Exception:
log.warning("outbox_cleanup_skipped")
async def pending_watch() -> None:
"""Admission control input and queue metrics (depth, oldest job age, unpublished outbox), once per second."""
while True:
try:
async with db.connection() as conn:
queue = await jobs.queue_stats(conn)
state["pending_jobs"] = queue["queued"] + queue["processing"]
state["queue"] = queue
except Exception:
state["queue"] = None # unknown is not zero: the gauge disappears instead of lying
await asyncio.sleep(config.PENDING_CHECK_SECONDS)
async def reconciler_loop() -> None:
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
while True:
try:
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
log.info("reconcile %s", result)
except Exception:
log.warning("reconcile_skipped: dependency unavailable")
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
async def readiness_watch() -> None:
failures = 0
while True:
if await db.ready():
failures = 0
if not state["ready"]:
state["ready"] = True
log.info("ready_again: database reachable")
else:
failures += 1
if failures >= config.READY_FAILURES and state["ready"]:
state["ready"] = False
log.warning("not_ready: database unreachable, business requests get 503")
s3_ok = await run_in_threadpool(s3.probe)
if s3_ok != state["s3_ok"]:
state["s3_ok"] = s3_ok
log.info("object_storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
await asyncio.sleep(config.READY_CHECK_SECONDS)
@asynccontextmanager
async def lifespan(_app: FastAPI):
previous = {}
def on_signal(signum, frame):
if not state["draining"]:
state["draining"] = True
log.info("draining signal=%s: new requests get 503", signal.Signals(signum).name)
handler = previous.get(signum)
if callable(handler):
handler(signum, frame)
for sig in (signal.SIGTERM, signal.SIGINT):
previous[sig] = signal.getsignal(sig)
signal.signal(sig, on_signal)
await db.open()
telemetry.register_pool(db)
telemetry.register_queue(state)
await sessions.open()
await run_in_threadpool(s3.open)
state["s3_ok"] = await run_in_threadpool(s3.probe)
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()),
asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())]
log.info(
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
)
log.info("pipeline_config pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s",
config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION)
yield
for task in tasks:
task.cancel()
await publisher.close()
await sessions.close()
await db.close()
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
telemetry.shutdown()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
@app.middleware("http")
async def guard(request: Request, call_next):
rid = uuid.uuid4().hex
request.state.rid = rid
# X-Request-Id <-> trace: the id is a span attribute and is present in every log line of the request.
telemetry.set_attributes(messenger__request_id=rid, messenger__instance_id=config.INSTANCE_ID)
exempt = request.url.path in EXEMPT_PATHS
if state["draining"] and request.url.path != "/health/live":
count_rejection("draining")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
if not exempt and not state["ready"]:
count_rejection("not_ready")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
count_rejection("inflight")
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
state["inflight"] += 1
try:
fault = request.headers.get("x-fault", "") if config.FAULT_INJECTION else ""
if fault:
await inject_http_fault(fault)
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
except asyncio.TimeoutError:
count_rejection("request_timeout")
log.warning("request_timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
response = error_body(
rid, config.INSTANCE_ID, 503, "unavailable",
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
)
except Exception as exc:
telemetry.record_exception(exc)
log.exception("unhandled_error request_id=%s", rid)
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
finally:
state["inflight"] -= 1
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
async def inject_http_fault(fault: str) -> None:
"""Test stand only (FAULT_INJECTION=1): `X-Fault: slow` adds 600 ms inside the handler path, `X-Fault: error`
raises an unexpected exception (-> 500). Used by the track-1 load to create rare slow/failed requests whose
trace survival can be counted (span attribute messenger.fault)."""
telemetry.set_attributes(messenger__fault=fault[:16])
if fault == "slow":
await asyncio.sleep(0.6)
elif fault == "error":
raise RuntimeError("injected handler failure")
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
if exc.status in (413, 429, 503):
count_rejection(exc.code)
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
def parse_uuid(value: object) -> Optional[str]:
if not isinstance(value, str):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def non_blank(value: object, max_len: int) -> bool:
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
return False
return value.translate(ZERO_WIDTH).strip() != ""
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
declared = request.headers.get("content-length")
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise too_large
raw = bytearray()
async for chunk in request.stream():
raw += chunk
if len(raw) > config.MAX_BODY_BYTES:
raise too_large
try:
body = json.loads(bytes(raw).decode("utf-8"))
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Некорректный JSON") from None
if not isinstance(body, dict):
raise bad("Ожидается JSON-объект")
allowed = set(required) | set(optional)
if set(body) - allowed:
raise bad("Лишние поля в запросе")
if set(required) - set(body):
raise bad("Отсутствуют обязательные поля")
return body
def ensure_capacity() -> None:
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
def bearer_token(request: Request) -> str:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
token = value.strip()
if scheme.lower() != "bearer" or not token:
raise unauthorized()
return token
async def actor(request: Request) -> dict:
user = await sessions.resolve(bearer_token(request))
if user is None:
raise unauthorized()
return user
def basic_credentials(request: Request) -> Tuple[str, str]:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized("Basic")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized("Basic") from None
username, separator, password = decoded.partition(":")
if not separator:
raise unauthorized("Basic")
return username, password
async def member_chat(user: dict, chat_id: str) -> dict:
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
async with db.connection() as conn:
chat = await repo.chat_for_member(conn, cid, user["id"])
if chat is None:
raise not_found()
return chat
def page_params(request: Request) -> Tuple[int, Optional[str]]:
query = request.query_params
limit = 50
if "limit" in query:
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
raise bad("limit должен быть целым 1..100")
limit = int(query["limit"])
cursor = query.get("cursor")
if cursor is not None and cursor == "":
raise bad("Некорректный cursor")
return limit, cursor
def keyset(scope: str, cursor: Optional[str]):
if cursor is None:
return None
try:
return decode_cursor(scope, cursor)
except BadCursor:
raise bad("Некорректный cursor") from None
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
return JSONResponse({"items": items, "next_cursor": next_cursor})
def detect_content_type(head: bytes) -> Optional[str]:
if head[:8] == PNG_MAGIC:
return "image/png"
if head[:3] == JPEG_MAGIC:
return "image/jpeg"
return None
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
raise bad("filename: 1..255 символов, не только пробелы")
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
raise bad("filename содержит недопустимые символы")
@app.get("/health/live")
async def live():
return {"status": "ok"}
@app.get("/health/ready")
async def ready():
if not await db.ready():
raise unavailable("База данных недоступна")
if not state["s3_ok"]:
raise unavailable("Хранилище файлов недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
body = await read_json(request, ("username", "display_name", "password"))
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
if not non_blank(body["display_name"], 100):
raise bad("display_name: 1..100 символов, не только пробелы")
password = body["password"]
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
ensure_capacity()
password_hash = await run_in_threadpool(passwords.hash_password, password)
async with db.connection() as conn:
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
if user is None:
raise ApiError(409, "conflict", "username уже занят")
log.info("user_created id=%s", user["id"])
return JSONResponse(user, status_code=201)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
if await request.body():
raise bad("POST /auth/sessions не принимает тело запроса")
username, password = basic_credentials(request)
row = None
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
async with db.connection() as conn:
row = await repo.credentials(conn, username)
stored = row["password_hash"] if row else passwords.DUMMY_HASH
matches = await run_in_threadpool(passwords.verify_password, stored, password)
if row is None or not matches:
log.info("login_failed username_known=%s", row is not None)
raise unauthorized("Basic")
user = repo.public_user(row)
token, expires_at = await sessions.create(user)
log.info("session_created user=%s", user["id"])
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request):
token = bearer_token(request)
user = await sessions.resolve(token)
if user is None:
raise unauthorized()
await sessions.revoke(token)
log.info("session_revoked user=%s", user["id"])
return Response(status_code=204)
@app.get("/api/v1/users/me")
async def me(request: Request):
return await actor(request)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await actor(request)
body = await read_json(request, ("title", "member_ids"))
if not non_blank(body["title"], 100):
raise bad("title: 1..100 символов, не только пробелы")
raw_ids = body["member_ids"]
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
raise bad("member_ids: массив из 1..100 UUID")
ids: List[str] = []
for value in raw_ids:
parsed = parse_uuid(value)
if parsed is None:
raise bad("member_ids содержит некорректный UUID")
ids.append(parsed)
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user["id"] not in ids:
raise bad("Создатель должен входить в member_ids")
ensure_capacity()
async def insert(conn):
return await repo.create_chat(conn, body["title"], ids)
try:
chat = await db.in_transaction(insert)
except repo.UnknownMember:
raise bad("member_ids содержит неизвестного пользователя") from None
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
return JSONResponse(chat, status_code=201)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = page_params(request)
scope = "chats:" + user["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
return page(scope, items, has_more)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, request: Request):
return await member_chat(await actor(request), chat_id)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
user = await actor(request)
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
idem_key = request.headers.get("idempotency-key", "")
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
body = await read_json(request, ("text",), ("attachment_ids",))
text = body["text"]
attachment_ids_raw = body.get("attachment_ids", [])
if not isinstance(attachment_ids_raw, list):
raise bad("attachment_ids должен быть массивом UUID")
if len(attachment_ids_raw) > 10:
raise bad("attachment_ids: максимум 10 вложений")
attachment_ids: List[str] = []
for value in attachment_ids_raw:
parsed = parse_uuid(value)
if parsed is None:
raise bad("attachment_ids содержит некорректный UUID")
attachment_ids.append(parsed)
if len(set(attachment_ids)) != len(attachment_ids):
raise bad("attachment_ids содержит дубликаты")
has_text = isinstance(text, str) and non_blank(text, 4000)
has_attachments = len(attachment_ids) > 0
if not has_text and not has_attachments:
raise bad("text: 1..4000 символов, не только пробелы")
if isinstance(text, str) and len(text) > 4000:
raise bad("text: 1..4000 символов, не только пробелы")
if not isinstance(text, str):
raise bad("text должен быть строкой")
ensure_capacity()
normalized = {"text": text}
if attachment_ids:
normalized["attachment_ids"] = attachment_ids
body_hash = hashlib.sha256(
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
async def do(conn):
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
status, message = await db.in_transaction(do)
if status == "conflict":
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
if status == "not_member":
raise not_found()
if status == "attachment_not_found":
raise not_found()
if status == "attachment_failed":
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
return JSONResponse(message, status_code=201)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request):
chat = await member_chat(await actor(request), chat_id)
limit, cursor = page_params(request)
scope = "messages:" + chat["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
return page(scope, items, has_more)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request):
user = await actor(request)
chat = await member_chat(user, chat_id)
filename = request.query_params.get("filename", "")
validate_filename(filename)
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared_ct not in ALLOWED_CONTENT_TYPES:
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
declared_length = request.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
raise too_large
ensure_capacity()
# Backpressure: do not accept more work than the pipeline can hold; the client retries later.
if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS:
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
sha = hashlib.sha256()
size = 0
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
head = b""
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
if len(head) < 8:
head += chunk[: 8 - len(head)]
if len(head) == 8 and detect_content_type(head) is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
sha.update(chunk)
spool.write(chunk)
if size == 0:
raise bad("Пустой файл")
actual_ct = detect_content_type(head)
if actual_ct is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
if actual_ct != declared_ct:
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
async with upload_slots:
await run_in_threadpool(inspect_image, spool, size, actual_ct)
attachment_id = str(uuid.uuid4())
staging_key = "staging/" + attachment_id
object_key = "attachments/%s/original" % attachment_id
async with db.connection() as conn:
await repo.begin_attachment(
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(),
staging_key, object_key,
)
try:
async with upload_slots:
await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct)
except BaseException:
defer(compensate(attachment_id, staging_key))
raise
# Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox.
try:
att = await db.in_transaction(lambda conn: repo.queue_attachment(
conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS))
except BaseException:
await compensate(attachment_id, staging_key)
raise
if att == repo.QUEUE_FULL:
defer(compensate(attachment_id, staging_key))
count_rejection("queue_full")
raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"})
if att is None:
raise unavailable("Загрузка отменена сверкой, повторите запрос")
telemetry.set_attributes(messenger__attachment_id=att["id"])
if faults.active(filename, "api_crash_after_commit"):
faults.crash("API after COMMIT of job/outbox, before publish")
relay.notify()
log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size)
return JSONResponse(att, status_code=202)
def defer(coro) -> None:
task = asyncio.ensure_future(coro)
background.add(task)
task.add_done_callback(background.discard)
async def compensate(attachment_id: str, object_key: str) -> None:
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
try:
async with db.connection() as conn:
await repo.abort_attachment(conn, attachment_id)
await run_in_threadpool(s3.delete, object_key)
except BaseException:
log.warning("compensation_incomplete attachment=%s, left for reconciler", attachment_id)
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
user = await actor(request)
aid = parse_uuid(attachment_id)
if aid is None:
raise bad("attachment_id должен быть UUID")
async with db.connection() as conn:
row = await repo.get_attachment(conn, aid)
if row is None:
raise not_found()
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
if chat is None:
raise not_found()
return row
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variants = []
if row["status"] == "ready":
async with db.connection() as conn:
variants = await repo.get_variants(conn, row["id"], row["pipeline_version"])
return repo._attachment(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variant = request.query_params.get("variant", "original")
if variant not in VARIANTS:
raise bad("variant: original, thumbnail, crop или optimized")
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing")
else "Обработка вложения завершилась ошибкой")
key, content_type = row["object_key"], row["content_type"]
if variant != "original":
async with db.connection() as conn:
found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant]
if not found:
raise not_found()
key, content_type = found[0]["object_key"], found[0]["content_type"]
opened = await run_in_threadpool(s3.open_stream, key)
if opened is None:
log.error("object_missing attachment=%s variant=%s", row["id"], variant)
raise unavailable("Файл временно недоступен")
chunks, length = opened
return StreamingResponse(
iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)},
)
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def fallback(request: Request):
raise not_found()
# Last statement on purpose: instrument_app() must run AFTER @app.middleware("http") is registered, so the OTel
# ASGI middleware is the outermost user middleware and guard's own 429/503/500 answers are in spans and metrics.
telemetry.setup("messenger-api", app)
+100
View File
@@ -0,0 +1,100 @@
"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work).
Rows are locked while they are published; `published_at` is set in the same transaction only after the
broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are
expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish
leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS.
"""
import asyncio
import logging
from opentelemetry.trace import SpanKind
from app import config, telemetry
log = logging.getLogger("messenger.outbox")
class Relay:
def __init__(self, db, publisher) -> None:
self.db = db
self.publisher = publisher
self.wake = asyncio.Event()
self.broker_ok = False
self.published = 0
def notify(self) -> None:
self.wake.set()
async def run_once(self) -> int:
sent, failure = 0, None
async with self.db.connection() as conn:
async with conn.transaction():
cur = await conn.execute(
"SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()"
" ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED",
(config.OUTBOX_BATCH,),
)
for row in await cur.fetchall():
try:
await self.publish(row["payload"])
except Exception as exc:
# Keep what was already confirmed: commit those rows, record the error, stop the batch.
await conn.execute(
"UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s",
(type(exc).__name__, row["id"]),
)
failure = exc
break
await conn.execute(
"UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1"
" WHERE id = %s",
(row["id"],),
)
sent += 1
self.published += sent
if failure is not None:
raise failure
return sent
async def publish(self, payload: dict) -> None:
"""PRODUCER span, child of the context stored with the job (upload request or failed attempt);
the message then carries the producer's own traceparent, so the consumer becomes its child."""
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "publish",
"messaging.destination.name": config.BROKER_EXCHANGE, "messaging.message.id": payload["job_id"],
"messenger.job.attempt": payload["attempt"], "messenger.attachment_id": payload["attachment_id"]}
with telemetry.span(config.BROKER_QUEUE + " publish", kind=SpanKind.PRODUCER,
context=telemetry.extract(payload), attributes=attrs):
body = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")}
body.update(telemetry.inject())
await self.publisher.publish(body)
async def loop(self) -> None:
delay = config.OUTBOX_POLL_SECONDS
while True:
try:
await asyncio.wait_for(self.wake.wait(), delay)
except asyncio.TimeoutError:
pass
self.wake.clear()
try:
while await self.run_once() == config.OUTBOX_BATCH:
pass
if not self.broker_ok:
log.info("outbox_relay_resumed: broker reachable")
self.broker_ok = True
delay = config.OUTBOX_POLL_SECONDS
except Exception as exc:
if self.broker_ok:
log.warning("outbox_relay_paused: %s; jobs stay in outbox", type(exc).__name__)
self.broker_ok = False
await self.publisher.close()
delay = min(10.0, max(delay * 2, 1.0))
async def cleanup(self) -> int:
async with self.db.connection() as conn:
cur = await conn.execute(
"DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)",
(config.OUTBOX_RETENTION_SECONDS,),
)
return cur.rowcount
+16
View File
@@ -0,0 +1,16 @@
from argon2 import PasswordHasher
from argon2.exceptions import Argon2Error, VerifyMismatchError
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
def hash_password(password: str) -> str:
return HASHER.hash(password)
def verify_password(stored: str, password: str) -> bool:
try:
return HASHER.verify(stored, password)
except (VerifyMismatchError, Argon2Error, ValueError):
return False
+155
View File
@@ -0,0 +1,155 @@
"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout).
Key layout (all keys are generated by the server):
staging/<attachment_id> written by the API before 202, read by the worker
attachments/<attachment_id>/original server-side copy made by the worker
attachments/<attachment_id>/v<N>/<variant>.* variants of pipeline version N
Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row.
Cases (all gated by a grace period so in-flight work is never touched):
stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both
orphan_object object without any row/variant row, older than grace -> delete object
staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete
staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete
(after that a replay is refused; the attachment stays failed and visible)
missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing
Queued/processing work is never touched: its staging object is the input of a live job.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
from starlette.concurrency import run_in_threadpool
from app import config
log = logging.getLogger("messenger.reconcile")
LOCK_ID = 2026_0104
PREFIXES = ("staging/", "attachments/")
BATCH = 500
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
started = time.monotonic()
result = {
"dry_run": dry_run, "grace_seconds": grace_seconds,
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0,
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
}
async with db.connection() as conn:
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
if not (await cur.fetchone())["got"]:
return {"skipped": "another reconciler is running"}
try:
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace_seconds)
await _stale_uploading(conn, s3, cutoff, dry_run, result)
await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result)
await _orphan_objects(conn, s3, cutoff, dry_run, result)
await _missing_objects(conn, s3, cutoff, dry_run, result)
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
result["seconds"] = round(time.monotonic() - started, 3)
return result
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, coalesce(staging_key, object_key) AS key FROM attachments"
" WHERE status = 'uploading' AND created_at < %s",
(cutoff,),
)
for row in await cur.fetchall():
modified = await run_in_threadpool(s3.head, row["key"])
if modified is not None and modified >= cutoff:
result["protected_fresh_objects"] += 1
continue
result["stale_uploading"] += 1
if dry_run:
continue
if modified is not None:
await run_in_threadpool(s3.delete, row["key"])
result["deleted_objects"] += 1
done = await conn.execute(
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
)
result["removed_rows"] += done.rowcount
async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL"
" AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
(retention_cutoff,),
)
for row in await cur.fetchall():
if await run_in_threadpool(s3.head, row["staging_key"]) is None:
continue
result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1
if dry_run:
continue
again = await conn.execute(
"SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))",
(row["id"], retention_cutoff),
)
if await again.fetchone() is not None:
await run_in_threadpool(s3.delete, row["staging_key"])
result["deleted_objects"] += 1
async def _known_keys(conn, keys):
cur = await conn.execute(
"SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)"
" UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)"
" UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)",
(keys, keys, keys),
)
return {row["k"] for row in await cur.fetchall()}
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
for prefix in PREFIXES:
pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix)))
while True:
page = await run_in_threadpool(next, pages, None)
if page is None:
break
result["objects_scanned"] += len(page)
keys = [o["Key"] for o in page]
if not keys:
continue
known = await _known_keys(conn, keys)
for obj in page:
if obj["Key"] in known or obj["LastModified"] >= cutoff:
continue
result["orphan_objects"] += 1
if not dry_run and not await _known_keys(conn, [obj["Key"]]):
await run_in_threadpool(s3.delete, obj["Key"])
result["deleted_objects"] += 1
async def _missing_objects(conn, s3, cutoff, dry_run, result):
last = None
while True:
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
(cutoff, last, last, BATCH),
)
rows = await cur.fetchall()
if not rows:
return
for row in rows:
result["rows_scanned"] += 1
if await run_in_threadpool(s3.exists, row["object_key"]):
continue
result["missing_objects"] += 1
if not dry_run:
done = await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
)
result["marked_failed"] += done.rowcount
last = str(rows[-1]["id"])
+338
View File
@@ -0,0 +1,338 @@
import hashlib
import json
from typing import List, Optional, Tuple
import psycopg
from app import config, jobs
from app.cursor import fmt
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
CHAT_QUERY = (
"SELECT c.id, c.title, c.created_at,"
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
)
class UnknownMember(Exception):
pass
def decoded(value) -> str:
return bytes(value).decode("utf-8")
def public_user(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": decoded(row["display_name"]),
"created_at": fmt(row["created_at"]),
}
def _chat(row) -> dict:
return {
"id": str(row["id"]),
"title": decoded(row["title"]),
"member_ids": [str(value) for value in row["member_ids"]],
"created_at": fmt(row["created_at"]),
}
def _message(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": decoded(row["text"]),
"created_at": fmt(row["created_at"]),
}
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
msg = _message(row)
msg["attachment_ids"] = attachment_ids
return msg
VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2}
def _variant(row) -> dict:
return {
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
def _attachment(row, variants=None) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [],
"error_code": row["error_code"],
"created_at": fmt(row["created_at"]),
"updated_at": fmt(row["updated_at"]),
}
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
try:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
" RETURNING id, username, display_name, created_at",
(username, display_name.encode("utf-8"), password_hash),
)
except psycopg.errors.UniqueViolation:
return None
return public_user(await cur.fetchone())
async def credentials(conn, username: str):
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,),
)
return await cur.fetchone()
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_at)"
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
" RETURNING id, title, created_at",
(title.encode("utf-8"),),
)
chat = await cur.fetchone()
cur = await conn.execute(
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
(chat["id"], member_ids),
)
if cur.rowcount != len(member_ids):
raise UnknownMember()
chat["member_ids"] = list(member_ids)
return _chat(chat)
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
cur = await conn.execute(
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id),
)
row = await cur.fetchone()
return _chat(row) if row else None
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
CHAT_QUERY
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
" ORDER BY c.created_at, c.id LIMIT %s",
(user_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
return [_chat(row) for row in rows[:limit]], len(rows) > limit
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
)
row = await cur.fetchone()
return _message(row) if row else None
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
cur = await conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
)
rows = await cur.fetchall()
return [str(row["attachment_id"]) for row in rows]
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
for pos, aid in enumerate(attachment_ids):
await conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, aid, pos),
)
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
" ORDER BY created_at, id LIMIT %s",
(chat_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
messages = []
for row in rows[:limit]:
aids = await _fetch_message_attachment_ids(conn, row["id"])
messages.append(_message_with_attachments(row, aids))
return messages, len(rows) > limit
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
raw = "%s:%s:%s" % (user_id, chat_id, key)
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
attachment_ids: List[str], idem_key: str, body_hash: str):
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
)
if await cur.fetchone() is None:
return "not_member", None
cur = await conn.execute(
"SELECT request_body_hash, response_json FROM idempotency_keys"
" WHERE user_id = %s AND chat_id = %s AND key = %s",
(sender_id, chat_id, idem_key),
)
existing = await cur.fetchone()
if existing:
if existing["request_body_hash"] != body_hash:
return "conflict", None
return "replay", json.loads(existing["response_json"])
if attachment_ids:
cur = await conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
(attachment_ids,),
)
att_rows = await cur.fetchall()
found_ids = {str(row["id"]) for row in att_rows}
for aid in attachment_ids:
if aid not in found_ids:
return "attachment_not_found", None
for row in att_rows:
if str(row["chat_id"]) != chat_id:
return "attachment_not_found", None
if row["status"] == "failed":
return "attachment_failed", None
text_bytes = text.encode("utf-8") if text else b""
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text_bytes, chat_id, sender_id),
)
row = await cur.fetchone()
if row is None:
return "not_member", None
if attachment_ids:
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
message = _message_with_attachments(row, list(attachment_ids))
await conn.execute(
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
)
return "created", message
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
(ttl_seconds,),
)
return cur.rowcount
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" staging_key, status, error_code, pipeline_version, created_at, updated_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())",
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key,
config.PIPELINE_VERSION),
)
ATTACHMENT_COLUMNS = (
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status,"
" error_code, pipeline_version, created_at, updated_at"
)
QUEUE_FULL = "queue_full"
ADMISSION_LOCK = 2026_0105
async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0):
"""uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff).
With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock,
so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight.
"""
if max_pending:
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,))
if await jobs.pending_count(conn) >= max_pending:
return QUEUE_FULL
cur = await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS,
(attachment_id,),
)
row = await cur.fetchone()
if row is None:
return None
await jobs.enqueue(conn, attachment_id, actor)
return _attachment(row)
async def abort_attachment(conn, attachment_id: str) -> None:
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
async def get_attachment(conn, attachment_id: str):
cur = await conn.execute(
"SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'",
(attachment_id,),
)
return await cur.fetchone()
async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]:
cur = await conn.execute(
"SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants"
" WHERE attachment_id = %s AND pipeline_version = %s",
(attachment_id, pipeline_version),
)
rows = await cur.fetchall()
return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99))
+122
View File
@@ -0,0 +1,122 @@
import logging
from typing import Iterator, Optional, Tuple
import boto3
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.errors import unavailable
log = logging.getLogger("messenger.s3")
MISSING = ("NoSuchKey", "404", "NotFound")
class S3Storage:
def __init__(self) -> None:
self.client = None
self.probe_client = None
def open(self) -> None:
self.probe_client = self._client(1.0, 1.5)
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
def _client(self, connect: float, read: float):
return boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY_ID,
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
config=BotoConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"max_attempts": 0},
),
)
def probe(self) -> bool:
try:
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
try:
fileobj.seek(0)
self.client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
)
except (BotoCoreError, ClientError) as exc:
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
except BotoCoreError as exc:
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
body = response["Body"]
return body.iter_chunks(65536), int(response["ContentLength"])
def head(self, key: str):
"""LastModified of the object, or None when it does not exist."""
try:
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
raise
def exists(self, key: str) -> bool:
return self.head(key) is not None
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
def list_pages(self, prefix: str):
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
yield page.get("Contents", [])
# ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient)
def put_file(self, key: str, path: str, content_type: str) -> None:
with open(path, "rb") as handle:
self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type)
def download(self, key: str, path: str) -> bool:
"""False when the object does not exist."""
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return False
raise
with open(path, "wb") as handle:
for chunk in response["Body"].iter_chunks(1 << 20):
handle.write(chunk)
return True
def copy(self, source_key: str, target_key: str) -> None:
"""Server-side copy: the original bytes never pass through the worker again."""
self.client.copy_object(
Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key},
)
def ready(self) -> bool:
try:
self.client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
+154
View File
@@ -0,0 +1,154 @@
import hashlib
import json
import logging
import secrets
from datetime import datetime, timedelta, timezone
from typing import Optional
import valkey.asyncio as valkey
from app import config, repo
from app.cursor import fmt
from app.errors import unavailable
log = logging.getLogger("messenger.sessions")
REVOKED = "revoked"
class SessionStore:
def __init__(self, db) -> None:
self.db = db
self.client = None
self.stats = {"hits": 0, "misses": 0, "errors": 0}
async def open(self) -> None:
if not config.CACHE_URL:
log.info("session cache disabled: CACHE_URL is empty")
return
self.client = valkey.Valkey.from_url(
config.CACHE_URL,
socket_timeout=config.CACHE_TIMEOUT,
socket_connect_timeout=config.CACHE_TIMEOUT,
decode_responses=True,
)
async def close(self) -> None:
if self.client is not None:
await self.client.aclose()
self.client = None
@staticmethod
def digest(token: str) -> bytes:
return hashlib.sha256(token.encode("utf-8")).digest()
@staticmethod
def key(digest: bytes) -> str:
return "session:" + digest.hex()
async def create(self, user: dict) -> tuple:
token = secrets.token_urlsafe(32)
digest = self.digest(token)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
async with self.db.connection() as conn:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
" VALUES (%s, %s, clock_timestamp(), %s)",
(digest, user["id"], expires_at),
)
await self._cache_put(digest, user, expires_at)
return token, fmt(expires_at)
async def resolve(self, token: str) -> Optional[dict]:
digest = self.digest(token)
cached = await self._cache_get(digest)
if cached == REVOKED:
self.stats["hits"] += 1
return None
if cached is not None:
self.stats["hits"] += 1
return cached
self.stats["misses"] += 1
async with self.db.connection() as conn:
cur = await conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
" FROM sessions s JOIN users u ON u.id = s.user_id"
" WHERE s.token_hash = %s AND s.expires_at > now()",
(digest,),
)
row = await cur.fetchone()
if row is None:
return None
user = repo.public_user(row)
await self._cache_put(digest, user, row["expires_at"])
return user
async def revoke(self, token: str) -> None:
digest = self.digest(token)
await self._cache_revoke(digest)
try:
async with self.db.connection() as conn:
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
except Exception:
await self._cache_release(digest)
raise
async def sweep(self) -> int:
async with self.db.connection() as conn:
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
return cur.rowcount
def _cache_ttl(self, expires_at: datetime) -> int:
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
async def _cache_get(self, digest: bytes) -> Optional[dict]:
if self.client is None:
return None
try:
raw = await self.client.get(self.key(digest))
except Exception:
self._cache_failed("get")
return None
if not raw:
return None
if raw == REVOKED:
return REVOKED
try:
return json.loads(raw)
except ValueError:
return None
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
if self.client is None:
return
ttl = self._cache_ttl(expires_at)
if ttl <= 0:
return
payload = json.dumps(user)
try:
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
except Exception:
self._cache_failed("set")
async def _cache_release(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.delete(self.key(digest))
except Exception:
self._cache_failed("release")
async def _cache_revoke(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
except Exception:
self._cache_failed("revoke")
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
def _cache_failed(self, operation: str) -> None:
self.stats["errors"] += 1
if self.stats["errors"] % 50 == 1:
log.warning("session_cache_failed %s, falling back to postgres (errors=%d)", operation, self.stats["errors"])
+393
View File
@@ -0,0 +1,393 @@
"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
"""
import json
import logging
import os
import re
import resource
import sys
import time
from contextlib import contextmanager
from typing import Optional
from opentelemetry import context as otel_context
from opentelemetry import metrics, propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import config
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
QUEUE = config.BROKER_QUEUE
JOB_TYPE = "attachment.process.v1"
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
tracer = trace.get_tracer("messenger")
meter = metrics.get_meter("messenger")
_state = {"service": "messenger", "providers": []}
# ---------------------------------------------------------------------------------------------- logging
class EventFilter(logging.Filter):
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
def filter(self, record: logging.LogRecord) -> bool:
if not getattr(record, "event", None):
match = self.EVENT_RE.match(str(record.msg))
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
return True
class JsonFormatter(logging.Formatter):
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
def format(self, record: logging.LogRecord) -> str:
out = {
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
"level": record.levelname,
"service": _state["service"],
"event": getattr(record, "event", ""),
"logger": record.name,
"msg": record.getMessage(),
}
ctx = trace.get_current_span().get_span_context()
if ctx.is_valid:
out["trace_id"] = format(ctx.trace_id, "032x")
out["span_id"] = format(ctx.span_id, "016x")
if record.exc_info:
out["exc"] = self.formatException(record.exc_info)
return json.dumps(out, ensure_ascii=False)
def _setup_logging(logger_provider=None) -> None:
root = logging.getLogger()
for handler in list(root.handlers):
root.removeHandler(handler)
root.setLevel(logging.INFO)
stdout = logging.StreamHandler(sys.stdout)
stdout.setFormatter(JsonFormatter())
stdout.addFilter(EventFilter())
root.addHandler(stdout)
# The exporter's own warnings (backend down) must not be fed back into the exporter.
sdk_log = logging.getLogger("opentelemetry")
sdk_log.propagate = False
sdk_log.addHandler(stdout)
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
logging.getLogger(noisy).setLevel(logging.WARNING)
if logger_provider is not None:
from opentelemetry.sdk._logs import LoggingHandler
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otlp.addFilter(EventFilter())
root.addHandler(otlp)
# ---------------------------------------------------------------------------------------------- sampling
def _build_sampler():
"""ParentBased(root=KindFilter(TraceIdRatio)).
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
Children always follow the parent's decision, so an upload and its worker job are sampled together.
"""
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
TraceIdRatioBased)
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
decisions = meter.create_counter(
"messenger.telemetry.sampler.decisions", unit="{span}",
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
class KindFilter(Sampler):
def __init__(self, delegate):
self.delegate = delegate
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
return SamplingResult(Decision.DROP)
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
trace_state)
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
return result
def get_description(self):
return "KindFilter{%s}" % self.delegate.get_description()
class CountingParentBased(ParentBased):
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
parent = trace.get_current_span(parent_context).get_span_context()
if parent.is_valid:
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
"rule": "parent"})
return result
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
class _CountingExporter:
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
def __init__(self, delegate):
from opentelemetry.sdk.trace.export import SpanExportResult
self.delegate = delegate
self.ok = SpanExportResult.SUCCESS
self.counter = meter.create_counter(
"messenger.telemetry.spans.exported", unit="{span}",
description="Spans handed to the OTLP exporter by outcome (success/failure)")
def export(self, spans):
result = self.delegate.export(spans)
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
return result
def shutdown(self):
return self.delegate.shutdown()
def force_flush(self, timeout_millis: int = 30000):
return self.delegate.force_flush(timeout_millis)
# ---------------------------------------------------------------------------------------------- setup
def setup(service_name: str, fastapi_app=None) -> None:
_state["service"] = service_name
if not ENABLED:
_setup_logging(None)
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
return
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
res = Resource.create({
"service.name": service_name,
"service.namespace": "messenger",
"service.version": SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
})
views = [
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
View(instrument_name="http.server.request.duration",
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.attachment.time_to_ready",
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
# No catch-all View("*"): an instrument matched by two views would be exported twice.
# Instruments without a matching view get the SDK default aggregation.
]
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
views=views)
metrics.set_meter_provider(meter_provider)
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
trace.set_tracer_provider(tracer_provider)
logger_provider = LoggerProvider(resource=res)
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
_setup_logging(logger_provider)
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
# placeholders), so values, message bodies and hashes never reach db.query.text.
BotocoreInstrumentor().instrument()
if fastapi_app is not None:
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
exclude_spans=["receive", "send"])
_process_metrics()
logging.getLogger("messenger.telemetry").info(
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
def shutdown() -> None:
for provider in _state["providers"]:
try:
provider.shutdown()
except Exception:
pass
# ---------------------------------------------------------------------------------------------- context carrier
def inject() -> dict:
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
carrier: dict = {}
if trace.get_current_span().get_span_context().is_valid:
propagate.inject(carrier)
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
def extract(carrier: Optional[dict]):
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
return otel_context.Context()
clean = {"traceparent": str(carrier["traceparent"])}
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
clean["tracestate"] = carrier["tracestate"]
return propagate.extract(clean, context=otel_context.Context())
def set_attributes(**attrs) -> None:
span = trace.get_current_span()
for key, value in attrs.items():
if value is not None:
span.set_attribute(key.replace("__", "."), value)
@contextmanager
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
record_exception=True, set_status_on_exception=True) as current:
yield current
def record_exception(exc: BaseException) -> None:
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
current = trace.get_current_span()
current.record_exception(exc)
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
def mark_error(current, description: str) -> None:
current.set_status(Status(StatusCode.ERROR, description))
# ---------------------------------------------------------------------------------------------- instruments
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
def _rss_bytes() -> int:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE
except OSError:
return 0
def _process_metrics() -> None:
from opentelemetry.metrics import Observation
def cpu(_options):
own = resource.getrusage(resource.RUSAGE_SELF)
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
description="Resident set size of the process")
def register_pool(db) -> None:
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def count(_options):
stats = db.pool.get_stats()
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
yield Observation(idle, {"db.client.connection.state": "idle"})
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
meter.create_observable_up_down_counter(
"db.client.connection.pending_requests",
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
meter.create_observable_up_down_counter(
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
def register_queue(state: dict) -> None:
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
"""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def depth(_options):
queue = state.get("queue")
if queue is None:
return []
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
def oldest(_options):
queue = state.get("queue")
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
description="Jobs accepted but not finished, by state")
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
description="Age of the oldest queued/processing job since durable acceptance")
def register_worker(worker) -> None:
if not ENABLED:
return
from opentelemetry.metrics import Observation
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
unit="{job}", description="Jobs being processed right now")
meter.create_observable_gauge("messenger.worker.capacity",
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
description="Concurrent job slots of this worker")
jobs_processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
job_duration = meter.create_histogram(
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
time_to_ready = meter.create_histogram(
"messenger.attachment.time_to_ready", unit="s",
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")
+367
View File
@@ -0,0 +1,367 @@
"""Attachment worker: a separate process consuming attachments.process (python -m app.worker).
Per delivery:
1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ
2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op
3. download staging object, verify size+sha256 against DB metadata
4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants
5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes)
6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner
7. delete staging (best effort, reconciler finishes it), then ack
Ack happens only after the durable effect (6) or after the failure/retry decision is committed.
A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases.
"""
import asyncio
import hashlib
import json
import logging
import os
import resource
import signal
import sys
import tempfile
import time
import uuid
from datetime import datetime, timezone
from opentelemetry.trace import SpanKind
from app import config, faults, jobs, telemetry
from app.broker import connect, declare
from app.db import Database
from app.s3 import S3Storage
log = logging.getLogger("messenger.worker")
PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"}
class Permanent(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class Transient(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail or code
class LeaseLost(Exception):
pass
def child_limits() -> None:
memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
cpu = config.JOB_TIMEOUT_SECONDS
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
os.nice(5)
def sha256_file(path: str):
digest, size = hashlib.sha256(), 0
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
size += len(chunk)
return digest.hexdigest(), size
class Worker:
def __init__(self) -> None:
self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8])
self.db = Database()
self.s3 = S3Storage()
self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY)
self.stopping = asyncio.Event()
self.inflight = 0
self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0}
# ------------------------------------------------------------------ lifecycle
async def run(self) -> None:
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
await self.db.open()
telemetry.register_pool(self.db)
telemetry.register_worker(self)
await asyncio.to_thread(self.s3.open)
connection = await connect()
channel = await connection.channel()
await channel.set_qos(prefetch_count=config.WORKER_PREFETCH)
_, queue = await declare(channel)
tag = await queue.consume(self.on_message, no_ack=False)
background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())]
log.info("worker_started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB",
self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS,
config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB)
await self.stopping.wait()
log.info("worker_stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight)
try:
await queue.cancel(tag)
except Exception:
pass
deadline = time.monotonic() + config.WORKER_GRACE_SECONDS
while self.inflight and time.monotonic() < deadline:
await asyncio.sleep(0.2)
for task in background:
task.cancel()
# Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed.
await connection.close()
await self.db.close()
log.info("worker_stopped stats=%s", self.stats)
async def alive(self) -> None:
while True:
try:
with open(config.WORKER_HEARTBEAT_FILE, "w") as handle:
handle.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(2)
async def sweeper(self) -> None:
while True:
await asyncio.sleep(config.JOB_SWEEP_SECONDS)
try:
swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner))
if swept:
log.warning("lease_expired recovered=%s", swept)
except Exception:
log.warning("lease_sweep_skipped: database unavailable")
# ------------------------------------------------------------------ delivery
async def on_message(self, message) -> None:
async with self.slots:
self.inflight += 1
started = time.monotonic()
# CONSUMER span: child of the PRODUCER span whose traceparent is in the job message (W3C carrier).
try:
carrier = json.loads(message.body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
carrier = None
if not isinstance(carrier, dict) or "traceparent" not in carrier:
carrier = dict(message.headers or {})
attrs = {"messaging.system": "rabbitmq", "messaging.operation.type": "process",
"messaging.destination.name": config.BROKER_QUEUE,
"messaging.message.id": str(message.message_id or ""),
"messaging.rabbitmq.message.redelivered": bool(message.redelivered)}
outcome = "requeued"
with telemetry.span(config.BROKER_QUEUE + " process", kind=SpanKind.CONSUMER,
context=telemetry.extract(carrier), attributes=attrs) as current:
try:
outcome = await self.handle(message)
except Exception:
log.exception("unexpected_error, message returned to queue")
await asyncio.sleep(1)
await self.settle(message.nack(requeue=True))
finally:
self.inflight -= 1
current.set_attribute("messenger.job.outcome", outcome)
if outcome in ("retry", "failed", "dlq", "requeued", "lost"):
telemetry.mark_error(current, outcome)
labels = {"job.type": telemetry.JOB_TYPE, "outcome": outcome}
telemetry.jobs_processed.add(1, labels)
telemetry.job_duration.record(time.monotonic() - started, labels)
@staticmethod
async def settle(awaitable) -> None:
try:
await awaitable
except Exception:
log.warning("could not settle delivery (connection lost); broker will redeliver")
async def handle(self, message) -> str:
"""Returns the delivery outcome: succeeded | retry | failed | lost | noop | dlq | requeued."""
msg = jobs.parse_message(message.body)
if msg is None:
self.stats["dlq"] += 1
log.error("job_invalid_message -> DLQ message_id=%s", message.message_id)
await self.settle(message.reject(requeue=False))
return "dlq"
telemetry.set_attributes(messenger__attachment_id=msg["attachment_id"], messenger__job__attempt=msg["attempt"])
try:
decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner))
except Exception:
log.warning("job_claim_failed: database unavailable, requeue job=%s", msg["job_id"])
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
if decision == "done" and job["state"] == "succeeded":
await self.cleanup_staging(job["attachment_id"])
if decision != "claimed":
self.stats["noop"] += 1
telemetry.set_attributes(messenger__job__decision=decision)
log.info("job_noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision,
message.redelivered)
await self.settle(message.ack())
return "noop"
started = time.monotonic()
log.info("job_claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"],
job["max_attempts"])
lease_lost = asyncio.Event()
keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost))
outcome = None
work = asyncio.create_task(self.process(job, message))
lost = asyncio.create_task(lease_lost.wait())
try:
done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2,
return_when=asyncio.FIRST_COMPLETED)
if work not in done:
work.cancel()
raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout")
outcome = "succeeded" if work.result() else "lost"
except LeaseLost:
outcome = "lost"
except Permanent as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=False)
except Transient as exc:
outcome = await self.record_failure(job, exc.code, exc.detail, transient=True)
except Exception as exc: # S3/DB/network errors
outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True)
finally:
keeper.cancel()
lost.cancel()
if outcome is None:
# Could not record the decision: keep the message; the lease expires and the sweeper takes over.
await asyncio.sleep(2)
await self.settle(message.nack(requeue=True))
return "requeued"
self.stats[outcome] = self.stats.get(outcome, 0) + 1
log.info("job_finished job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome,
time.monotonic() - started)
await self.settle(message.ack())
return outcome
async def cleanup_staging(self, attachment_id) -> None:
"""Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup."""
try:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, attachment_id)
if src and src["status"] == "ready" and src["staging_key"]:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", attachment_id)
async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None:
interval = max(1.0, config.JOB_LEASE_SECONDS / 3)
while True:
await asyncio.sleep(interval)
try:
alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner))
except Exception:
continue # DB blip: keep trying until the lease really expires
if not alive:
log.warning("lease_lost job=%s: another worker owns it now", job_id)
lease_lost.set()
return
async def record_failure(self, job, code: str, detail: str, transient: bool):
log.warning("job_attempt_failed job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code,
transient)
try:
return await self.db.in_transaction(
lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient))
except Exception:
log.warning("job_failure_not_recorded job=%s: database unavailable", job["id"])
return None
# ------------------------------------------------------------------ the actual work
async def process(self, job, message) -> bool:
async with self.db.connection() as conn:
src = await jobs.attachment_source(conn, job["attachment_id"])
if src is None or not src["staging_key"]:
raise Permanent("staging_missing", "attachment has no staging object")
filename = src["filename"]
first_delivery = job["attempt"] == 1 and not message.redelivered
if faults.active(filename, "worker_hang") and job["attempt"] == 1:
log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5))
time.sleep(config.JOB_LEASE_SECONDS * 2.5)
if faults.active(filename, "permanent"):
raise Permanent("invalid_image", "injected permanent failure")
if job["attempt"] <= faults.transient_until(filename):
raise Transient("dependency_unavailable", "injected transient failure")
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
source = os.path.join(tmp, "source")
if not await asyncio.to_thread(self.s3.download, src["staging_key"], source):
raise Permanent("staging_missing", "staging object not found")
digest, size = await asyncio.to_thread(sha256_file, source)
if digest != src["sha256"] or size != src["size_bytes"]:
raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload")
out_dir = os.path.join(tmp, "out")
os.mkdir(out_dir)
with telemetry.span("imageproc", attributes={"messenger.pipeline_version": job["pipeline_version"]}):
produced = await self.run_child(source, out_dir)
variants = []
prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"])
await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"])
for item in produced:
digest, size = await asyncio.to_thread(sha256_file, item["file"])
key = prefix + os.path.basename(item["file"])
await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"])
variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"],
"size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest})
if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS):
raise Permanent("processing_failed", "not all variants were produced")
if first_delivery and faults.active(filename, "worker_crash_after_put"):
faults.crash("after S3 put, before result COMMIT")
committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants))
if not committed:
log.warning("job_result_not_committed job=%s: lease lost, effect belongs to the new owner", job["id"])
return False
telemetry.time_to_ready.record(max(0.0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()),
{"job.type": telemetry.JOB_TYPE})
if first_delivery and faults.active(filename, "worker_crash_after_commit"):
faults.crash("after result COMMIT, before ack")
try:
await asyncio.to_thread(self.s3.delete, src["staging_key"])
except Exception:
log.warning("staging_cleanup_deferred attachment=%s", job["attachment_id"])
return True
async def run_child(self, source: str, out_dir: str) -> list:
env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app",
"JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)}
proc = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits,
)
try:
stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None
except asyncio.CancelledError:
proc.kill()
raise
code = proc.returncode
if code == 0:
return json.loads(stdout)
if code in PERMANENT_EXIT:
raise Permanent(PERMANENT_EXIT[code])
if code == -signal.SIGXCPU:
raise Permanent("image_too_complex", "CPU limit exceeded")
if code == -signal.SIGKILL:
raise Transient("worker_resource_exhausted", "child killed (OOM)")
raise Permanent("processing_failed", "child exit %s" % code)
def main() -> None:
telemetry.setup("messenger-worker")
try:
asyncio.run(Worker().run())
finally:
telemetry.shutdown()
if __name__ == "__main__":
main()
+460
View File
@@ -0,0 +1,460 @@
name: mtusi-lab6
# OpenTelemetry SDK settings shared by the API replicas and the worker (track 1 overrides them from the shell).
x-otel-env: &otel-env
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_EXPORTER_OTLP_TIMEOUT: "5"
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
OTEL_SEMCONV_STABILITY_OPT_IN: http
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-10000}
OTEL_BSP_MAX_QUEUE_SIZE: "2048"
OTEL_BSP_SCHEDULE_DELAY: "2000"
OTEL_BLRP_MAX_QUEUE_SIZE: "2048"
SERVICE_VERSION: "1.6.0"
x-api: &api-defaults
build:
context: .
image: mtusi-messenger:lab6
env_file:
- .env
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
- broker_url
depends_on:
rabbitmq:
condition: service_healthy
postgres:
condition: service_healthy
valkey:
condition: service_healthy
minio-init:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
interval: 5s
timeout: 3s
retries: 3
start_period: 15s
restart: unless-stopped
services:
nginx:
image: nginx:1.27.3-alpine
command: ["/bin/sh", "/opt/nginx-health.sh"]
environment:
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
HEALTH_INTERVAL: "1"
HEALTH_FALL: "2"
HEALTH_RISE: "2"
ports:
- "127.0.0.1:8080:8080"
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
depends_on:
api-1:
condition: service_healthy
api-2:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 64M
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"]
interval: 5s
timeout: 3s
retries: 3
start_period: 5s
restart: unless-stopped
api-1:
<<: *api-defaults
environment:
<<: *otel-env
INSTANCE_ID: api-1
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://minio:9000
BROKER_URL_FILE: /run/secrets/broker_url
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
api-2:
<<: *api-defaults
environment:
<<: *otel-env
INSTANCE_ID: api-2
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://minio:9000
BROKER_URL_FILE: /run/secrets/broker_url
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
worker:
build:
context: .
image: mtusi-messenger:lab6
command: ["python", "-m", "app.worker"]
env_file:
- .env
environment:
<<: *otel-env
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: http://minio:9000
CACHE_URL_FILE: ""
INSTANCE_ID: ""
DB_POOL_MAX: "6"
# Overridable from the shell for the track-1 experiment (shell env wins over .env interpolation).
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
- broker_url
depends_on:
rabbitmq:
condition: service_healthy
postgres:
condition: service_healthy
minio-init:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
stop_grace_period: 30s
deploy:
replicas: ${WORKER_REPLICAS:-2}
resources:
limits:
cpus: "${WORKER_CPUS:-2.00}"
memory: 1536M
healthcheck:
test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""]
interval: 5s
timeout: 3s
retries: 3
start_period: 15s
restart: unless-stopped
rabbitmq:
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
hostname: rabbitmq
entrypoint: ["/bin/sh", "/opt/entrypoint.sh"]
secrets:
- rabbitmq_password
volumes:
- rabbitmqdata:/var/lib/rabbitmq
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
- ./infra/rabbitmq/entrypoint.sh:/opt/entrypoint.sh:ro
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 20
start_period: 20s
restart: unless-stopped
postgres:
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_INITDB_ARGS: --data-checksums
command:
- postgres
- -c
- synchronous_commit=on
- -c
- shared_buffers=128MB
- -c
- work_mem=8MB
- -c
- max_connections=50
- -c
- track_io_timing=on
secrets:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql/data
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
restart: unless-stopped
valkey:
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
command:
- valkey-server
- --save
- ""
- --appendonly
- "no"
- --maxmemory
- 64mb
- --maxmemory-policy
- allkeys-lru
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
start_period: 5s
restart: unless-stopped
minio:
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
command: ["server", "/data"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
MINIO_BROWSER: "off"
secrets:
- minio_root_user
- minio_root_password
volumes:
- miniodata:/data
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "mc ready local || exit 1"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
restart: unless-stopped
minio-init:
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
entrypoint: ["/bin/sh", "/opt/init.sh"]
environment:
S3_BUCKET: messenger-attachments
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infra/minio/init.sh:/opt/init.sh:ro
- ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro
depends_on:
minio:
condition: service_healthy
restart: "no"
migrate:
build:
context: .
image: mtusi-messenger:lab6
command: ["python", "scripts/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
restart: "no"
# ------------------------------------------------------------------ observability (lab6)
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0@sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917
command:
- --config=/etc/otelcol/collector.yaml
- --config=/etc/otelcol/profile-${COLLECTOR_PROFILE:-default}.yaml
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
- ./infra/otel/profile-default.yaml:/etc/otelcol/profile-default.yaml:ro
- ./infra/otel/profile-tail.yaml:/etc/otelcol/profile-tail.yaml:ro
# OTLP 4317/4318 and self-metrics 8888 stay inside the compose network.
deploy:
resources:
limits:
cpus: "1.00"
memory: 256M
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.2.1@sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --web.enable-otlp-receiver
- --web.enable-lifecycle
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- promdata:/prometheus
ports:
- "127.0.0.1:9090:9090" # query API for evidence scripts (localhost only)
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
loki:
image: grafana/loki:3.4.2@sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
ports:
- "127.0.0.1:3100:3100"
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.1@sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
ports:
- "127.0.0.1:3200:3200"
deploy:
resources:
limits:
cpus: "1.00"
memory: 1024M
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
restart: unless-stopped
grafana:
image: grafana/grafana:11.5.2@sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
secrets:
- grafana_admin_password
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
ports:
- "127.0.0.1:3000:3000"
depends_on:
prometheus:
condition: service_healthy
loki:
condition: service_started
tempo:
condition: service_started
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 30
start_period: 15s
restart: unless-stopped
volumes:
pgdata:
miniodata:
rabbitmqdata:
promdata:
lokidata:
tempodata:
grafanadata:
secrets:
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
broker_url:
file: ./secrets/broker_url
rabbitmq_password:
file: ./secrets/rabbitmq_password
grafana_admin_password:
file: ./secrets/grafana_admin_password
+244
View File
@@ -0,0 +1,244 @@
# Панели dashboard «Messenger — наблюдаемость (lab6)»
Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. `$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).
## SLO (окно 30 мин) и алерты
**Доступность (SLO 99%)** (stat, единица: доля (0..1))
— 1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.
```promql
messenger:slo_availability:ratio30m
```
**Латентность ≤500 ms (SLO 95%)** (stat, единица: доля (0..1))
— Доля запросов в bucket le=0.5 s за 30 мин.
```promql
messenger:slo_latency:ratio30m
```
**Error ratio 5xx (2m)** (stat, единица: доля (0..1))
```promql
messenger:http_error_ratio:rate2m
```
**Старейшее задание** (stat, единица: секунды)
```promql
max(messenger_queue_oldest_age_seconds{queue="attachments.process"})
```
**Алерты Prometheus (firing/pending)** (alertlist)
Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).
**RPS API** (stat, единица: запросы/с)
```promql
messenger:http_requests:rate2m
```
**p95 API (5m)** (stat, единица: секунды)
```promql
messenger:http_latency_p95:rate5m
```
**Загрузка worker** (stat, единица: доля (0..1))
```promql
messenger:worker_saturation:ratio
```
**Ожидают пул БД** (stat, единица: штуки)
```promql
max(db_client_connection_pending_requests)
```
## API (messenger-api, все реплики)
**RPS по маршруту и методу** (timeseries, единица: запросы/с)
— Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID.
```promql
sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # {{http_request_method}} {{http_route}}
```
**RPS по классу статуса** (timeseries, единица: запросы/с)
```promql
sum by (status_class) (label_replace(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval]), "status_class", "${1}xx", "http_response_status_code", "(\\d)..")) # {{status_class}}
```
**Латентность API p50/p95/p99 (из histogram buckets)** (timeseries, единица: секунды)
— histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик.
```promql
histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # p99
```
**Доля ошибок 5xx** (timeseries, единица: доля (0..1))
— Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом.
```promql
(sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[$__rate_interval])) # 5xx / все
```
**p95 по маршруту** (timeseries, единица: секунды)
```promql
histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[$__rate_interval]))) # {{http_route}}
```
**Запросы в обработке** (timeseries, единица: штуки)
```promql
sum by (instance) (http_server_active_requests{service_name="messenger-api"}) # {{instance}}
```
## Очередь и worker
**Jobs по исходу (в сек.)** (timeseries, единица: операции/с)
— succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1
```promql
sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval])) # {{outcome}}
```
**Длительность job (succeeded) p50/p95/p99** (timeseries, единица: секунды)
```promql
histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[$__rate_interval]))) # p99
```
**Глубина очереди (из PostgreSQL)** (timeseries, единица: штуки)
— max() по репликам API: каждая публикует одно и то же значение из БД.
```promql
max by (queue, state) (messenger_queue_depth) # {{queue}} {{state}}
```
**Возраст старейшей job** (timeseries, единица: секунды)
— От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.
```promql
max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) # oldest age
```
**Время до ready p50/p95/p99** (timeseries, единица: секунды)
— messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready.
```promql
histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p50
```
```promql
histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p95
```
```promql
histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))) # p99
```
**Насыщение worker: занято / ёмкость** (timeseries, единица: штуки)
```promql
sum(messenger_worker_busy) # busy
```
```promql
sum(messenger_worker_capacity) # capacity
```
```promql
messenger:worker_saturation:ratio # saturation
```
## Ресурсы: пул БД, CPU, RAM
**Пул БД: used / idle / waiting** (timeseries, единица: штуки)
```promql
sum by (service_name, db_client_connection_state) (db_client_connection_count) # {{service_name}} {{db_client_connection_state}}
```
```promql
sum by (service_name) (db_client_connection_pending_requests) # {{service_name}} waiting
```
```promql
sum by (service_name) (db_client_connection_max) # {{service_name}} max
```
**CPU процессов (ядра)** (timeseries, единица: доля (0..1))
— worker: self + завершённые дочерние процессы обработки изображений.
```promql
sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval])) # {{service_name}} {{instance}}
```
```promql
rate(otelcol_process_cpu_seconds_total[$__rate_interval]) # otel-collector
```
**RSS процессов** (timeseries, единица: байты)
```promql
process_memory_usage_bytes # {{service_name}} {{instance}}
```
```promql
otelcol_process_memory_rss_bytes # otel-collector
```
**Telemetry pipeline: принято / отправлено / ошибки (items/s)** (timeseries, единица: штуки)
```promql
sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval])) # accepted spans
```
```promql
sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval])) # accepted log records
```
```promql
sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval])) # accepted metric points
```
```promql
sum by (exporter) (rate({__name__=~"otelcol_exporter_sent_(spans|log_records|metric_points)_total"}[$__rate_interval])) # sent {{exporter}}
```
```promql
sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0) # send failed spans
```
```promql
sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0) # dropped spans (queue full)
```
```promql
sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0) # send failed logs
```
```promql
sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0) # dropped logs (queue full)
```
```promql
sum by (exporter) (otelcol_exporter_queue_size) # queue {{exporter}}
```
## Логи и трассы
**Логи API и worker (warning/error)** (logs)
```
{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"
```
**Трассы с ошибками или медленнее 500 ms (TraceQL)** (table)
```
{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}
```
+90
View File
@@ -0,0 +1,90 @@
# Runbooks алертов (lab6)
Алерты вычисляет Prometheus (`infra/prometheus/rules/messenger.yml`) каждые 15 с. Состояние смотрится в Grafana
(dashboard «Messenger — наблюдаемость», панель «Алерты Prometheus») или командой:
```sh
curl -s http://127.0.0.1:9090/api/v1/alerts | python3 -m json.tool
```
Доставка во внешние каналы не настроена (по заданию достаточно локального firing/resolved).
Общий порядок диагностики: **метрика → трасса → лог**. Метрика говорит «что и сколько», трасса — «где в
цепочке тратится время / где ошибка», лог — «какое событие и с какими техническими ID».
Все шаги ниже используют только сохранённую telemetry.
---
## MessengerApiHighErrorRate
**Условие:** `messenger:http_error_ratio:rate2m > 0.05` при `messenger:http_requests:rate2m > 0.2`, `for: 1m`.
Числитель — ответы 5xx, знаменатель — все бизнес-запросы messenger-api (health-пробы не инструментированы).
4xx (неверный пароль, чужой чат, 409) в числитель не входят.
1. Dashboard → «RPS по классу статуса» и «Доля ошибок 5xx»: какой код (500 или 503) и с какого момента.
2. Prometheus: на каком маршруте ошибки
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))`.
3. Отличить БД от handler:
- пул: `max(db_client_connection_pending_requests)`, `db_client_connection_count{db_client_connection_state="used"}`.
Ожидание пула > 0 → соединения заняты долгими запросами к БД;
- Tempo (Explore → Tempo → TraceQL):
`{resource.service.name="messenger-api" && span.db.system="postgresql" && status=error}` — если в трассе
db-span (`INSERT`/`SELECT`) занимает почти всё время запроса и завершился исключением
`LockNotAvailable`/`QueryCanceled`/`OperationalError` — причина в PostgreSQL (блокировка, перегрузка, недоступность);
- если server span длинный, а db-spans короткие (единицы ms) — медленный/падающий handler
(`{span.messenger.fault="slow"}` в учебном стенде даёт именно такую картину).
4. Loki: `{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"` — события `unhandled_error`
(500, есть stack trace в атрибуте), `not_ready` (БД недоступна), `request_timeout`.
По `X-Request-Id`, который получил клиент: `{service_name="messenger-api"} |= "<request_id>"` → поле `trace_id`
→ кнопка «Open trace».
5. Если виновата БД: `docker compose exec postgres psql -U messenger -d messenger -c "SELECT pid, state,
wait_event_type, now()-xact_start AS age, left(query,80) FROM pg_stat_activity WHERE state <> 'idle'"` —
найти блокирующую транзакцию; согласовать и выполнить `SELECT pg_terminate_backend(<pid>)`.
6. Восстановление: доля 5xx падает сразу, alert уходит в resolved, когда окно 2m очистится (~2–3 мин).
## MessengerQueueLagGrowing
**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60`, `for: 1m`.
Возраст считается от COMMIT задания (момент ответа 202) по данным PostgreSQL; публикуют все реплики API,
поэтому `max()`.
1. Dashboard → «Глубина очереди», «Jobs по исходу», «Насыщение worker».
- `sum(rate(messenger_jobs_processed_total[1m])) == 0` при растущей глубине → worker не работает;
- `messenger:worker_saturation:ratio ≈ 1` и jobs/s > 0 → worker не успевает (нужно больше реплик/конкурентности);
- много `outcome="retry"` → ошибки зависимостей (S3/DB), см. логи `job_attempt_failed`.
2. Tempo: найти upload по attachment_id `{span.messenger.attachment_id="<id>"}`. Если в трассе есть
PRODUCER `attachments.process publish`, но нет CONSUMER — сообщение в брокере, worker его не забрал.
Если нет и PRODUCER — outbox не опубликован: смотреть `max(messenger_queue_depth{queue="outbox"})` и лог
`outbox_relay_paused` (брокер недоступен).
3. Loki: `{service_name="messenger-worker"}` — последние события `worker_stopping`/`worker_stopped` или отсутствие
`job_*` событий; `{service_name="messenger-worker"} | event="job_attempt_failed"` — причины retry.
4. Действие: `docker compose ps worker`, `docker compose start worker` (или масштабирование
`docker compose up -d --scale worker=3`). Задания не теряются: они в PostgreSQL/outbox и в durable очереди.
5. Восстановление: backlog уходит за секунды, возраст падает до 0, alert resolved через ~15–30 с.
## MessengerApiLatencyHigh
**Условие:** p95 из buckets всех реплик за 5m > 1 s, `for: 2m`. Шаги — как в HighErrorRate, пункт 3:
по трассам `{resource.service.name="messenger-api" && duration > 1s}` определить, какой span занимает время
(db / S3 / handler).
## MessengerDbPoolWaiting
**Условие:** `max(db_client_connection_pending_requests) > 0`, `for: 1m`. Пул исчерпан: долгие запросы или
блокировки в БД (см. HighErrorRate, п. 3 и 5) либо пул мал для нагрузки (`DB_POOL_MAX`).
## MessengerWorkerSaturated
**Условие:** занято > 90% слотов всех worker 5 минут. Если очередь при этом растёт — масштабировать worker;
проверить `messenger_job_duration_seconds` p95 (не стали ли задания медленнее).
## TelemetryExportFailing
**Условие:** очередь exporter заполнена больше чем на 20% либо есть send_failed/enqueue_failed > 0, `for: 1m`.
Бизнес-API **не** затронут: SDK и collector имеют ограниченные очереди.
1. `docker compose ps tempo loki prometheus` — какой backend недоступен.
2. Панель «Telemetry pipeline»: `otelcol_exporter_queue_size` по exporter (ёмкость 100 батчей) и
`otelcol_exporter_enqueue_failed_*` — сколько данных уже отброшено.
3. `docker compose start tempo loki`. Данные, отброшенные при переполнении очереди, не восстанавливаются —
их объём виден в счётчиках и фиксируется в отчёте инцидента.
+6
View File
@@ -0,0 +1,6 @@
ALERTS{alertstate=firing} во время финального прогона инцидентов A/B/C (источник: Prometheus TSDB), UTC
MessengerApiHighErrorRate page ['14:37:00..14:39:00']
MessengerApiLatencyHigh ticket ['14:38:00..14:41:45']
MessengerDbPoolWaiting ticket ['14:36:30..14:37:00']
MessengerQueueLagGrowing page ['14:35:00..14:35:00']
TelemetryExportFailing ticket ['14:41:00..14:42:45']
+9
View File
@@ -0,0 +1,9 @@
# 2026-11-09T14:55:39Z clean copy of git files, steps of .github/workflows/stand.yml (-p lab6clean to keep the main stand volumes)
secrets ready in secrets/ (.env: present)
compose config OK
Container lab6clean-tempo-1 Healthy
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
grafana dashboards: ['Messenger — наблюдаемость (lab6)']
prometheus rules: 13
+92
View File
@@ -0,0 +1,92 @@
ok 1 series Доступность (SLO 99%) Доступность (SLO 99%)
ok 1 series Латентность ≤500 ms (SLO 95%) Латентность ≤500 ms (SLO 95%)
ok 1 series Error ratio 5xx (2m) Error ratio 5xx (2m)
ok 1 series Старейшее задание Старейшее задание
ok 1 series RPS API RPS API
ok 1 series p95 API (5m) p95 API (5m)
ok 1 series Загрузка worker Загрузка worker
ok 1 series Ожидают пул БД Ожидают пул БД
ok 6 series RPS по маршруту и методу {{http_request_method}} {{http_route}}
ok 2 series RPS по классу статуса {{status_class}}
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p50
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p95
ok 1 series Латентность API p50/p95/p99 (из histogram bucket p99
ok 1 series Доля ошибок 5xx 5xx / все
ok 5 series p95 по маршруту {{http_route}}
ok 2 series Запросы в обработке {{instance}}
ok 1 series Jobs по исходу (в сек.) {{outcome}}
ok 1 series Длительность job (succeeded) p50/p95/p99 p50
ok 1 series Длительность job (succeeded) p50/p95/p99 p95
ok 1 series Длительность job (succeeded) p50/p95/p99 p99
ok 3 series Глубина очереди (из PostgreSQL) {{queue}} {{state}}
ok 1 series Возраст старейшей job oldest age
ok 1 series Время до ready p50/p95/p99 p50
ok 1 series Время до ready p50/p95/p99 p95
ok 1 series Время до ready p50/p95/p99 p99
ok 1 series Насыщение worker: занято / ёмкость busy
ok 1 series Насыщение worker: занято / ёмкость capacity
ok 1 series Насыщение worker: занято / ёмкость saturation
ok 4 series Пул БД: used / idle / waiting {{service_name}} {{db_client_connection_state}}
ok 2 series Пул БД: used / idle / waiting {{service_name}} waiting
ok 2 series Пул БД: used / idle / waiting {{service_name}} max
ok 4 series CPU процессов (ядра) {{service_name}} {{instance}}
ok 1 series CPU процессов (ядра) otel-collector
ok 4 series RSS процессов {{service_name}} {{instance}}
ok 1 series RSS процессов otel-collector
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted spans
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted log records
ok 1 series Telemetry pipeline: принято / отправлено / ошибк accepted metric points
ok 3 series Telemetry pipeline: принято / отправлено / ошибк sent {{exporter}}
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed spans
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped spans (queue full)
ok 1 series Telemetry pipeline: принято / отправлено / ошибк send failed logs
ok 1 series Telemetry pipeline: принято / отправлено / ошибк dropped logs (queue full)
ok 3 series Telemetry pipeline: принято / отправлено / ошибк queue {{exporter}}
panels: 32, prometheus queries without data: 0
grafana prometheus ok 3 values Доступность (SLO 99%)
grafana prometheus ok 3 values Латентность ≤500 ms (SLO 95%)
grafana prometheus ok 3 values Error ratio 5xx (2m)
grafana prometheus ok 9 values Старейшее задание
grafana prometheus ok 3 values RPS API
grafana prometheus ok 3 values p95 API (5m)
grafana prometheus ok 9 values Загрузка worker
grafana prometheus ok 9 values Ожидают пул БД
grafana prometheus ok 18 values RPS по маршруту и методу
grafana prometheus ok 5 values RPS по классу статуса
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 3 values Латентность API p50/p95/p99 (из histogram bucket
grafana prometheus ok 3 values Доля ошибок 5xx
grafana prometheus ok 15 values p95 по маршруту
grafana prometheus ok 6 values Запросы в обработке
grafana prometheus ok 3 values Jobs по исходу (в сек.)
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 3 values Длительность job (succeeded) p50/p95/p99
grafana prometheus ok 27 values Глубина очереди (из PostgreSQL)
grafana prometheus ok 9 values Возраст старейшей job
grafana prometheus ok 3 values Время до ready p50/p95/p99
grafana prometheus ok 3 values Время до ready p50/p95/p99
grafana prometheus ok 3 values Время до ready p50/p95/p99
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
grafana prometheus ok 9 values Насыщение worker: занято / ёмкость
grafana prometheus ok 36 values Пул БД: used / idle / waiting
grafana prometheus ok 18 values Пул БД: used / idle / waiting
grafana prometheus ok 18 values Пул БД: used / idle / waiting
grafana prometheus ok 34 values CPU процессов (ядра)
grafana prometheus ok 8 values CPU процессов (ядра)
grafana prometheus ok 36 values RSS процессов
grafana prometheus ok 9 values RSS процессов
grafana prometheus ok 2 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 8 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 18 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 91 values Telemetry pipeline: принято / отправлено / ошибк
grafana prometheus ok 27 values Telemetry pipeline: принято / отправлено / ошибк
grafana loki ok 7 values Логи API и worker (warning/error)
grafana tempo ok 17 traces Трассы с ошибками или медленнее 500 ms (TraceQL) (datasource proxy)
via Grafana: queries without data or with errors: 0
+20
View File
@@ -0,0 +1,20 @@
17:41:17 tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=100 batches
17:41:31 + 14 s queue=9/100 collector_rss=68.7MiB dropped=0 refused=0
17:41:44 + 27 s queue=24/100 collector_rss=81.4MiB dropped=0 refused=0
17:41:58 + 41 s queue=39/100 collector_rss=102MiB dropped=0 refused=0
17:42:12 + 55 s queue=39/100 collector_rss=123.9MiB dropped=0 refused=0
17:42:26 + 69 s queue=55/100 collector_rss=133.1MiB dropped=0 refused=0
17:42:40 + 83 s queue=70/100 collector_rss=160.9MiB dropped=0 refused=0
17:42:54 + 97 s queue=85/100 collector_rss=161.1MiB dropped=0 refused=0
17:43:08 +111 s queue=100/100 collector_rss=164.1MiB dropped=1062 refused=0
17:43:22 +125 s queue=100/100 collector_rss=173.4MiB dropped=15664 refused=0
17:43:36 +139 s queue=99/100 collector_rss=177.4MiB dropped=16214 refused=4248
17:43:50 +153 s queue=99/100 collector_rss=184MiB dropped=16214 refused=5310
17:44:04 +167 s queue=99/100 collector_rss=185.5MiB dropped=16214 refused=6372
17:44:18 +181 s queue=99/100 collector_rss=205.9MiB dropped=16214 refused=6372
17:44:32 +195 s queue=100/100 collector_rss=204.4MiB dropped=24597 refused=12004
17:44:46 +209 s queue=100/100 collector_rss=204.1MiB dropped=25147 refused=16252
17:45:00 +223 s queue=100/100 collector_rss=203.9MiB dropped=25147 refused=16802
17:45:01 result: peak queue=100/100, dropped by full queue=25147 spans, refused by memory_limiter=16802 spans, peak RSS=205.9 MiB
17:45:01 API: requests=24686 ok=24686 unexpected_errors=0 p99=55.79 ms
PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected
+16
View File
@@ -0,0 +1,16 @@
# 2026-11-09T13:40:58Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 1)
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 1.997}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab6-api-1-1 Up 47 hours (healthy)
$ docker compose start api-1 api-2
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh (run 2)
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 0.903}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
mtusi-lab6-api-2-1 Up 41 seconds (healthy)
$ docker compose start api-1 api-2
FAILOVER_DONE
+195
View File
@@ -0,0 +1,195 @@
{
"A_worker_stopped": {
"alert_pending_s": 70.62756085395813,
"alert_firing_s": 131,
"age_at_firing_s": 126,
"drain_after_restart_s": 3.6,
"resolved_after_restart_s": 14,
"lost_jobs": 0
},
"B_db_lock": {
"alert_pending_s": 45.0726158618927,
"alert_firing_s": 105,
"error_ratio_at_firing": 0.707,
"p95_at_firing_s": 6.92,
"db_span_ms": 5001,
"request_ms": 5004,
"cause": "QueryCanceled",
"resolved_after_unlock_s": 135
},
"C1_backends_down": {
"outage_s": 192,
"api_requests": 1703,
"api_unexpected_errors": 0,
"collector_samples": [
{
"t": 14,
"collector_mem": "190.5MiB",
"cpu": "0.05%",
"queue_tempo": 6.0,
"queue_loki": 0.0
},
{
"t": 26,
"collector_mem": "190.5MiB",
"cpu": "0.07%",
"queue_tempo": 6.0,
"queue_loki": 0.0
},
{
"t": 38,
"collector_mem": "190.5MiB",
"cpu": "0.04%",
"queue_tempo": 14.0,
"queue_loki": 0.0
},
{
"t": 50,
"collector_mem": "191.5MiB",
"cpu": "0.58%",
"queue_tempo": 21.0,
"queue_loki": 0.0
},
{
"t": 62,
"collector_mem": "190.8MiB",
"cpu": "0.31%",
"queue_tempo": 29.0,
"queue_loki": 0.0
},
{
"t": 74,
"collector_mem": "190.7MiB",
"cpu": "0.02%",
"queue_tempo": 36.0,
"queue_loki": 0.0
},
{
"t": 86,
"collector_mem": "190.3MiB",
"cpu": "0.13%",
"queue_tempo": 36.0,
"queue_loki": 0.0
},
{
"t": 98,
"collector_mem": "190.3MiB",
"cpu": "0.01%",
"queue_tempo": 44.0,
"queue_loki": 0.0
},
{
"t": 110,
"collector_mem": "191.7MiB",
"cpu": "0.02%",
"queue_tempo": 51.0,
"queue_loki": 0.0
},
{
"t": 122,
"collector_mem": "192.7MiB",
"cpu": "0.37%",
"queue_tempo": 57.0,
"queue_loki": 0.0
},
{
"t": 134,
"collector_mem": "191.9MiB",
"cpu": "0.10%",
"queue_tempo": 64.0,
"queue_loki": 0.0
},
{
"t": 146,
"collector_mem": "192.4MiB",
"cpu": "0.38%",
"queue_tempo": 64.0,
"queue_loki": 0.0
},
{
"t": 158,
"collector_mem": "192.4MiB",
"cpu": "0.22%",
"queue_tempo": 72.0,
"queue_loki": 0.0
},
{
"t": 170,
"collector_mem": "192MiB",
"cpu": "0.03%",
"queue_tempo": 79.0,
"queue_loki": 0.0
}
],
"alert": "firing",
"dropped_items": 0.0,
"send_failed_items": 387.0,
"traces_after_restart": 20,
"alert_resolved_after_restart_s": 35
},
"C2_collector_down": {
"outage_s": 60,
"api1_mem_before": "108.6MiB",
"api1_mem_during": [
"108.8MiB",
"109.3MiB",
"109.3MiB",
"109.1MiB",
"109.3MiB",
"119.3MiB"
],
"api_requests": 740,
"api_unexpected_errors": 0
},
"traffic": {
"profile": {
"rate_rps": 10,
"duration_s": 3600,
"upload_rate_rps": 0,
"slow_pct": 2,
"error_pct": 0,
"threads": 48,
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"
},
"started_at": 1794061933.7705631,
"finished_at": 1794062670.181464,
"generator_backlog_at_end": 0,
"requests": 7365,
"ok": 6335,
"successful_rps": 1.76,
"status_counts": {
"200": 1593,
"201": 4742,
"503": 816,
"502": 214
},
"server_errors_unexpected": 1030,
"latency_ms_plain_ok": {
"p50": 14.87,
"p95": 22.77,
"p99": 27.78,
"max": 8280.75
},
"faults_sent": {
"slow": 157,
"error": 0
},
"by_kind": {
"post_message": {
"n": 5510,
"p50_ms": 14.32,
"p99_ms": 25.09
},
"get_history": {
"n": 1855,
"p50_ms": 17.84,
"p99_ms": 43.91
},
"upload": {
"n": 0,
"p50_ms": null,
"p99_ms": null
}
}
}
}
+58
View File
@@ -0,0 +1,58 @@
17:32:13 background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)
17:32:43 === A. worker stopped (queue lag)
17:32:44 worker stopped at 17:32:44
17:32:44 3 uploads accepted (202) while worker is down: ['62859e73', '99403faf', '4ad2b534']
17:33:54 MessengerQueueLagGrowing pending (+71 s)
17:34:54 MessengerQueueLagGrowing firing (+131 s)
17:34:54 diagnosis from telemetry:
17:34:54 metrics: oldest job age=126 s, depth={'processing': 0.0, 'queued': 3.0}, worker jobs/s over 1m=0.000
17:34:54 logs: worker job_* events after stop: 0; last worker events: ['worker_stopped', 'worker_stopped']
17:34:54 trace of upload 62859e73: span kinds ['PRODUCER', 'SERVER'] -> published, no CONSUMER: the job waits in the queue
17:34:57 worker started (+131 s)
17:34:58 backlog drained: all 3 ready 3.6 s after worker start
17:35:08 MessengerQueueLagGrowing resolved (+14 s)
17:35:08 === B. DB lock on messages (slow/unavailable DB for writes and history reads)
17:35:08 ACCESS EXCLUSIVE lock taken on table messages at 17:35:08
17:35:53 MessengerApiHighErrorRate pending (+45 s)
17:36:53 MessengerApiHighErrorRate firing (+105 s)
17:36:53 diagnosis from telemetry:
17:36:53 metrics: 5xx ratio=70.7%, p95=6.92 s, pool used(max replica)=10 waiting=19
17:36:53 5xx/s /api/v1/chats 500 0.00
17:36:53 5xx/s /api/v1/chats/{chat_id}/messages 503 9.59
17:36:53 trace 23982edb8bd1246afc6bd6522faaa524 (failed request):
api SERVER POST /api/v1/chats/{chat_id}/messages 5004.2 ms ERROR {"messenger.request_id": "0f42d95e28624ecb9e443dc88c491177", "messenger.instance_id": "api-1", "http.response.status_code": "503"}
api CLIENT SELECT 0.1 ms UNSET {"db.system": "postgresql"}
api CLIENT SELECT chat_members 0.1 ms UNSET {"db.system": "postgresql"}
api CLIENT SELECT idempotency_keys 0.1 ms UNSET {"db.system": "postgresql"}
api CLIENT INSERT messages 5001.0 ms ERROR {"db.system": "postgresql"}
17:36:53 => DB span INSERT messages took 5001 ms of 5004 ms request, status ERROR, exception QueryCanceled: the time is spent in PostgreSQL
17:36:53 contrast: slow HANDLER trace 47ee6988aff06d8a0f5592a97fb9a967: request 615 ms, longest DB span 1.1 ms, all DB 8.3 ms
17:36:53 logs: 29 API warning/error records in 5 min, events=['not_ready', 'outbox_relay_paused']
17:36:53 operator step (pg_stat_activity) finds the blocker: 4246|active|Timeout|BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT
17:36:53 lock released (+106 s)
17:39:09 MessengerApiHighErrorRate resolved (+135 s)
17:39:09 === C1. Tempo and Loki stopped (telemetry backends down)
17:39:22 + 14 s collector mem=190.5MiB cpu=0.05% queue tempo=6 loki=0 (capacity 100 batches)
17:39:35 + 26 s collector mem=190.5MiB cpu=0.07% queue tempo=6 loki=0 (capacity 100 batches)
17:39:47 + 38 s collector mem=190.5MiB cpu=0.04% queue tempo=14 loki=0 (capacity 100 batches)
17:39:59 + 50 s collector mem=191.5MiB cpu=0.58% queue tempo=21 loki=0 (capacity 100 batches)
17:40:10 + 62 s collector mem=190.8MiB cpu=0.31% queue tempo=29 loki=0 (capacity 100 batches)
17:40:23 + 74 s collector mem=190.7MiB cpu=0.02% queue tempo=36 loki=0 (capacity 100 batches)
17:40:35 + 86 s collector mem=190.3MiB cpu=0.13% queue tempo=36 loki=0 (capacity 100 batches)
17:40:47 + 98 s collector mem=190.3MiB cpu=0.01% queue tempo=44 loki=0 (capacity 100 batches)
17:40:59 +110 s collector mem=191.7MiB cpu=0.02% queue tempo=51 loki=0 (capacity 100 batches)
17:41:10 +122 s collector mem=192.7MiB cpu=0.37% queue tempo=57 loki=0 (capacity 100 batches)
17:41:23 +134 s collector mem=191.9MiB cpu=0.10% queue tempo=64 loki=0 (capacity 100 batches)
17:41:35 +146 s collector mem=192.4MiB cpu=0.38% queue tempo=64 loki=0 (capacity 100 batches)
17:41:47 +158 s collector mem=192.4MiB cpu=0.22% queue tempo=72 loki=0 (capacity 100 batches)
17:41:59 +170 s collector mem=192MiB cpu=0.03% queue tempo=79 loki=0 (capacity 100 batches)
17:41:59 alert TelemetryExportFailing during outage: firing
17:41:59 API during the outage: 1703 requests, unexpected errors=0; alert TelemetryExportFailing=firing
17:42:20 tempo/loki started and ready (+192 s)
17:42:30 new traces after restart visible: 20 (export resumed)
17:42:56 TelemetryExportFailing resolved (+35 s)
17:43:16 telemetry lost and counted: enqueue_failed(queue full)=0 items, send_failed(retries exhausted)=387 items
17:43:16 === C2. Collector stopped (SDK side buffering)
17:44:30 api-1 RSS before=108.6MiB during=['108.8MiB', '109.3MiB', '109.3MiB', '109.1MiB', '109.3MiB', '119.3MiB']; 740 requests, unexpected errors=0
17:44:30 summary written to /Users/alenchha/mtusi2026lab6/evidence/incidents.json
PASS: incidents A,B,C diagnosed from telemetry, alerts fired and resolved
+80
View File
@@ -0,0 +1,80 @@
17:36:11 scenario A: scenario_a
17:36:11 client saw: missing X-Request-Id
17:36:11 committed attachment 8f23222e-3018-4692-a4ce-f67bfcf43871 survived the API crash
17:36:12 statuses: queued -> ready
17:36:12 audit attempt=1 enqueued api:api-2
17:36:12 audit attempt=1 claimed 832caf57a286:a39a11c5
17:36:12 audit attempt=1 succeeded 832caf57a286:a39a11c5
17:36:13 scenario A PASS (2.4s)
17:36:13 scenario B: scenario_b
17:36:37 statuses: queued -> processing -> queued -> processing -> ready
17:36:37 audit attempt=1 enqueued api:api-1
17:36:37 audit attempt=1 claimed 18bdb5d574fa:8873b063
17:36:37 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:4f98f68 lease_expired delay=1.97s
17:36:37 audit attempt=2 claimed 832caf57a286:a39a11c5
17:36:37 audit attempt=2 succeeded 832caf57a286:a39a11c5
17:36:38 scenario B PASS (24.9s)
17:36:38 scenario C: scenario_c
17:36:39 statuses: queued -> ready
17:36:39 audit attempt=1 enqueued api:api-2
17:36:39 audit attempt=1 claimed 18bdb5d574fa:4f98f687
17:36:39 audit attempt=1 succeeded 18bdb5d574fa:4f98f687
17:36:49 scenario C PASS (10.4s)
17:36:49 scenario D: scenario_d
17:36:58 statuses: processing -> queued -> ready
17:36:58 audit attempt=1 enqueued api:api-1
17:36:58 audit attempt=1 claimed 832caf57a286:a39a11c5
17:36:58 audit attempt=1 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=1.25s
17:36:58 audit attempt=2 claimed 18bdb5d574fa:79c71044
17:36:58 audit attempt=2 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=2.58s
17:36:58 audit attempt=3 claimed 832caf57a286:a39a11c5
17:36:58 audit attempt=3 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=4.3s
17:36:58 audit attempt=4 claimed 18bdb5d574fa:79c71044
17:36:58 audit attempt=4 succeeded 18bdb5d574fa:79c71044
17:36:59 backoff delays: [1.25, 2.58, 4.3]
17:36:59 scenario D PASS (10.4s)
17:36:59 scenario E: scenario_e
17:37:00 audit attempt=1 enqueued api:api-1
17:37:00 audit attempt=1 claimed 832caf57a286:a39a11c5
17:37:00 audit attempt=1 failed 832caf57a286:a39a11c5 invalid_image
17:37:00 scenario E PASS (0.7s)
17:37:00 scenario F: scenario_f
17:37:24 failed registry contains the job; replay
17:37:47 statuses: queued -> processing -> ready
17:37:47 audit attempt=1 enqueued api:api-2
17:37:47 audit attempt=1 claimed 18bdb5d574fa:79c71044
17:37:47 audit attempt=1 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=1.99s
17:37:47 audit attempt=2 claimed 832caf57a286:a39a11c5
17:37:47 audit attempt=2 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=2.59s
17:37:47 audit attempt=3 claimed 18bdb5d574fa:79c71044
17:37:47 audit attempt=3 retry_scheduled 18bdb5d574fa:79c71044 dependency_unavailable delay=4.09s
17:37:47 audit attempt=4 claimed 832caf57a286:a39a11c5
17:37:47 audit attempt=4 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=11.17s
17:37:47 audit attempt=5 claimed 18bdb5d574fa:79c71044
17:37:47 audit attempt=5 failed 18bdb5d574fa:79c71044 retry_exhausted
17:37:47 audit attempt=6 replayed operator:test-suite
17:37:47 audit attempt=6 claimed 832caf57a286:a39a11c5
17:37:47 audit attempt=6 retry_scheduled 832caf57a286:a39a11c5 dependency_unavailable delay=19.41s
17:37:47 audit attempt=7 claimed 18bdb5d574fa:79c71044
17:37:47 audit attempt=7 succeeded 18bdb5d574fa:79c71044
17:37:49 second replay refused: job is succeeded, attachment is ready
17:37:50 scenario F PASS (50.4s)
17:37:50 scenario G: scenario_g
17:38:17 statuses: processing -> queued -> processing -> ready
17:38:17 audit attempt=1 enqueued api:api-1
17:38:17 audit attempt=1 claimed 832caf57a286:a39a11c5
17:38:17 audit attempt=1 retry_scheduled sweeper:18bdb5d574fa:79c7104 lease_expired delay=1.77s
17:38:17 audit attempt=2 claimed 18bdb5d574fa:79c71044
17:38:17 audit attempt=2 succeeded 18bdb5d574fa:79c71044
17:38:49 scenario G PASS (59.1s)
17:38:49 scenario H: scenario_h
17:38:53 DLQ depth 0 -> 1
17:38:53 scenario H PASS (3.8s)
17:38:53 scenario I: scenario_i
17:38:58 broker down: upload 202, outbox holds 1 unpublished job
17:39:03 statuses: queued -> ready
17:39:03 audit attempt=1 enqueued api:api-2
17:39:03 audit attempt=1 claimed 832caf57a286:a39a11c5
17:39:03 audit attempt=1 succeeded 832caf57a286:a39a11c5
17:39:04 scenario I PASS (11.4s)
PASS: failure windows A B C D E F G H I
+14
View File
@@ -0,0 +1,14 @@
17:26:48 stop all workers
17:26:49 upload 202 queued id=bc973dcb-254c-4a56-8746-5accb5e85267
17:26:49 message e459cc31-bf5f-431c-9c1d-e473e95a59a1 created with queued attachment
17:26:54 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0}
17:26:54 restart api-1 api-2 rabbitmq (volumes kept)
17:26:59 started before: mtusi-lab6-api-1-1 2026-11-09T14:18:47.635837802Z; mtusi-lab6-api-2-1 2026-11-09T14:18:47.722431011Z; mtusi-lab6-rabbitmq-1 2026-11-07T14:44:49.033246884Z
17:26:59 started after: mtusi-lab6-api-1-1 2026-11-09T14:26:55.137236792Z; mtusi-lab6-api-2-1 2026-11-09T14:26:55.0483775Z; mtusi-lab6-rabbitmq-1 2026-11-09T14:26:55.741984625Z
17:27:03 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0}
17:27:03 start workers
17:27:06 status sequence observed: queued -> ready
17:27:07 objects: ['attachments/bc973dcb-254c-4a56-8746-5accb5e85267/original', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/crop.jpg', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/optimized.webp', 'attachments/bc973dcb-254c-4a56-8746-5accb5e85267/v1/thumbnail.jpg']
17:27:07 redelivery: republish the finished job through the outbox (duplicate message)
17:27:14 duplicate acked as no-op: variants=3 successes=1
PASS: worker down / API+broker restart / redelivery no-op
+27
View File
@@ -0,0 +1,27 @@
09:16:45 message sent request_id=07f7f48bf1c74c01b3f546bd98c19492 (text marker kept private)
09:16:47 upload 5791d45a-87cb-465a-b8af-47a02fcfe006: queued -> queued -> ready
09:16:51 Tempo: trace ef3b4c403f734bcb04ee9b5ce085aa68, 42 spans; tree (DB client spans omitted):
api SERVER POST /api/v1/chats/{chat_id}/attachments 10.2 ms UNSET {"http.response.status_code": "202", "messenger.request_id": "f33563a1565d46bea878a83207f42bef", "messenger.instance_id": "api-1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
api CLIENT S3.PutObject 3.9 ms UNSET {"rpc.method": "PutObject"}
api PRODUCER attachments.process publish 1.5 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "1", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
worker CONSUMER attachments.process process 4.9 ms ERROR {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "1", "messenger.job.outcome": "retry"}
api PRODUCER attachments.process publish 4.6 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.job.attempt": "2", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006"}
worker CONSUMER attachments.process process 169.0 ms UNSET {"messaging.message.id": "e0be9d72-9d4a-4b43-9f2f-1765a668b857", "messenger.attachment_id": "5791d45a-87cb-465a-b8af-47a02fcfe006", "messenger.job.attempt": "2", "messenger.job.outcome": "succeeded"}
worker CLIENT S3.GetObject 6.8 ms UNSET {"rpc.method": "GetObject"}
worker INTERNAL imageproc 137.9 ms UNSET {"messenger.pipeline_version": "1"}
worker CLIENT S3.CopyObject 3.8 ms UNSET {"rpc.method": "CopyObject"}
worker CLIENT S3.PutObject 2.2 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.PutObject 1.8 ms UNSET {"rpc.method": "PutObject"}
worker CLIENT S3.DeleteObject 1.0 ms UNSET {"rpc.method": "DeleteObject"}
09:16:51 PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links
09:16:51 Loki: 6 records with trace_id=ef3b4c403f734bcb04ee9b5ce085aa68: [('api', 'attachment_queued'), ('worker', 'job_claimed'), ('worker', 'job_attempt_failed'), ('worker', 'job_finished'), ('worker', 'job_claimed'), ('worker', 'job_finished')]
09:16:51 Loki record fields: ['code_filepath', 'code_function', 'code_lineno', 'deployment_environment_name', 'detected_level', 'event', 'flags', 'observed_timestamp', 'scope_name', 'service_instance_id', 'service_name', 'service_namespace', 'service_version', 'severity_number', 'severity_text', 'span_id', 'telemetry_sdk_language', 'telemetry_sdk_name', 'telemetry_sdk_version', 'trace_id', 'ts']
09:16:55 HTTP 500 X-Request-Id=9302b0182c0243df8acc92865370b939 -> Loki event=unhandled_error trace_id=6a254a70c77d5419df95d6f8d2e681ce
09:16:55 Tempo: GET /api/v1/chats status=ERROR request_id matches; exception event recorded=True
09:16:55 message POST trace 26d27e3a639403ae1576707024358b6e: ['POST /api/v1/chats/{chat_id}/messages', 'SELECT', 'SELECT chat_members', 'SELECT idempotency_keys', 'INSERT messages', 'INSERT idempotency_keys']
09:16:55 PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from traces and logs
09:16:55 PASS: 994 series, label names: ['__name__', 'db_client_connection_state', 'decision', 'deployment_environment_name', 'error_type', 'http_request_method', 'http_response_status_code', 'http_route', 'instance', 'job', 'job_type', 'le', 'network_protocol_version', 'outcome', 'process_cpu_owner', 'queue', 'rule', 'service_instance_id', 'service_name', 'service_version', 'state', 'url_scheme']
09:16:55 http_route values (templates only): ['/api/v1/attachments/{attachment_id}', '/api/v1/auth/sessions', '/api/v1/chats', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users']
09:17:07 PASS: required metrics present: http_server_request_duration_seconds_bucket, messenger_jobs_processed_total, messenger_job_duration_seconds_bucket, messenger_attachment_time_to_ready_seconds_bucket, messenger_queue_depth, messenger_queue_oldest_age_seconds, messenger_worker_busy, messenger_worker_capacity, db_client_connection_count, db_client_connection_pending_requests, process_cpu_time_seconds_total, process_memory_usage_bytes, otelcol_receiver_accepted_spans_total, otelcol_exporter_sent_spans_total, otelcol_exporter_queue_size
PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)
+32
View File
@@ -0,0 +1,32 @@
# 2026-11-11T06:17:14Z scripts/restart.sh restarts business and telemetry stores, volumes kept
$ docker compose ps (before)
mtusi-lab6-api-1-1 Up 3 minutes (healthy)
mtusi-lab6-api-2-1 Up 3 minutes (healthy)
mtusi-lab6-grafana-1 Up 3 minutes (healthy)
mtusi-lab6-loki-1 Up 3 minutes (healthy)
mtusi-lab6-minio-1 Up 3 minutes (healthy)
mtusi-lab6-otel-collector-1 Up 3 minutes
mtusi-lab6-postgres-1 Up 3 minutes (healthy)
mtusi-lab6-prometheus-1 Up 3 minutes (healthy)
mtusi-lab6-rabbitmq-1 Up 3 minutes (healthy)
mtusi-lab6-tempo-1 Up 3 minutes (healthy)
mtusi-lab6-valkey-1 Up 3 minutes (healthy)
mtusi-lab6-worker-1 Up 3 minutes (healthy)
mtusi-lab6-worker-2 Up 3 minutes (healthy)
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
$ docker compose ps (after)
mtusi-lab6-api-1-1 Up 10 seconds (healthy)
mtusi-lab6-api-2-1 Up 10 seconds (healthy)
mtusi-lab6-grafana-1 Up 21 seconds (healthy)
mtusi-lab6-loki-1 Up 20 seconds (healthy)
mtusi-lab6-minio-1 Up 23 seconds (healthy)
mtusi-lab6-otel-collector-1 Up 21 seconds
mtusi-lab6-postgres-1 Up 22 seconds (healthy)
mtusi-lab6-prometheus-1 Up 21 seconds (healthy)
mtusi-lab6-rabbitmq-1 Up 21 seconds (healthy)
mtusi-lab6-tempo-1 Up 11 seconds (health: starting)
mtusi-lab6-valkey-1 Up 22 seconds (healthy)
mtusi-lab6-worker-1 Up 11 seconds (healthy)
mtusi-lab6-worker-2 Up 11 seconds (healthy)
+7
View File
@@ -0,0 +1,7 @@
# 2026-11-09T14:26:13Z
$ make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make validate
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
$ make test
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
+16
View File
@@ -0,0 +1,16 @@
# 2026-11-09T14:29:49Z restart of telemetry stores, volumes kept
mtusi-lab6-grafana-1 Up 2 days (healthy)
mtusi-lab6-loki-1 Up 2 days (healthy)
mtusi-lab6-prometheus-1 Up 2 days (healthy)
mtusi-lab6-tempo-1 Up 2 days (healthy)
Container mtusi-lab6-prometheus-1 Started
Container mtusi-lab6-grafana-1 Started
Container mtusi-lab6-tempo-1 Started
Container mtusi-lab6-loki-1 Started
mtusi-lab6-grafana-1 Up 27 seconds (healthy)
mtusi-lab6-loki-1 Up 25 seconds (healthy)
mtusi-lab6-prometheus-1 Up 27 seconds (healthy)
mtusi-lab6-tempo-1 Up 26 seconds (healthy)
Tempo: trace of 2026-11-07 incident B 23982edb8bd1246afc6bd6522faaa524 spans: 5
Loki: API warn/error records for that window: 48 streams
Prometheus: firing samples of MessengerApiHighErrorRate in that window: 9
+12
View File
@@ -0,0 +1,12 @@
{"config": "off", "rep": 1, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794231942.6489458, "finished_at": 1794232032.6500669, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4003, "202": 90, "200": 1335, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.43, "p95": 10.28, "p99": 31.67, "max": 77.52}, "faults_sent": {"slow": 56, "error": 62}, "by_kind": {"post_message": {"n": 4003, "p50_ms": 6.1, "p99_ms": 27.37}, "get_history": {"n": 1397, "p50_ms": 7.63, "p99_ms": 32.03}, "upload": {"n": 90, "p50_ms": 19.99, "p99_ms": 166.23}}}, "cpu_pct_avg": {"api-2": 9.01, "api-1": 9.09, "worker": 15.35, "otel-collector": 0.07, "tempo": 0.43, "nginx": 1.29, "grafana": 0.98, "postgres": 5.1, "rabbitmq": 6.69, "valkey": 0.55, "minio": 1.63, "loki": 0.57, "prometheus": 0.36}, "mem_mib_max": {"api-2": 102.9, "api-1": 93.6, "worker": 134.2, "otel-collector": 35.7, "tempo": 291.9, "nginx": 5.2, "grafana": 135.7, "postgres": 58.6, "rabbitmq": 153.1, "valkey": 12.4, "minio": 306.3, "loki": 82.9, "prometheus": 273.4}, "telemetry_delta": {"sdk_span_decisions_sample": -38.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -38.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 26288.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794231942.6489012, 1794232032.66341], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "on100", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232111.334157, "finished_at": 1794232201.333892, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4029, "202": 90, "200": 1321, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.63, "p95": 16.28, "p99": 54.35, "max": 484.0}, "faults_sent": {"slow": 60, "error": 50}, "by_kind": {"post_message": {"n": 4029, "p50_ms": 8.21, "p99_ms": 54.1}, "get_history": {"n": 1371, "p50_ms": 10.46, "p99_ms": 54.35}, "upload": {"n": 90, "p50_ms": 21.71, "p99_ms": 61.23}}}, "cpu_pct_avg": {"api-2": 11.5, "api-1": 11.33, "worker": 15.68, "otel-collector": 0.6, "tempo": 1.08, "nginx": 1.26, "grafana": 0.91, "postgres": 4.65, "rabbitmq": 6.94, "valkey": 0.49, "minio": 1.57, "loki": 0.55, "prometheus": 0.36}, "mem_mib_max": {"api-2": 111.2, "api-1": 115.5, "worker": 144.0, "otel-collector": 71.3, "tempo": 420.8, "nginx": 13.8, "grafana": 136.0, "postgres": 79.0, "rabbitmq": 141.6, "valkey": 12.0, "minio": 299.2, "loki": 92.9, "prometheus": 276.3}, "telemetry_delta": {"sdk_span_decisions_sample": 56361.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 56361.0, "sdk_spans_exported_failed": null, "col_spans_in": 57045.0, "col_logs_in": 340.0, "col_points_in": 1224.0, "col_spans_out": 57045.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21529255.0, "tempo_spans_in": 57045.0, "tempo_bytes_in": 21033821.0, "loki_bytes_in": 135765.0, "loki_lines_in": 340.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 60, "error": 50, "pipeline_consumer": 90}, "window": [1794232111.3341079, 1794232201.347548], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "head10", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232280.328288, "finished_at": 1794232370.6738691, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5435, "successful_rps": 60.39, "status_counts": {"201": 4045, "202": 90, "200": 1300, "500": 55}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 12.98, "p99": 39.25, "max": 71.67}, "faults_sent": {"slow": 68, "error": 55}, "by_kind": {"post_message": {"n": 4045, "p50_ms": 8.03, "p99_ms": 39.25}, "get_history": {"n": 1355, "p50_ms": 9.8, "p99_ms": 39.23}, "upload": {"n": 90, "p50_ms": 20.56, "p99_ms": 28.61}}}, "cpu_pct_avg": {"worker": 14.92, "api-2": 9.9, "api-1": 9.7, "otel-collector": 0.24, "tempo": 0.35, "nginx": 1.22, "grafana": 0.85, "postgres": 4.5, "rabbitmq": 6.47, "valkey": 0.51, "minio": 1.47, "loki": 0.5, "prometheus": 0.33}, "mem_mib_max": {"worker": 146.7, "api-2": 101.2, "api-1": 113.3, "otel-collector": 67.4, "tempo": 500.3, "nginx": 5.3, "grafana": 135.7, "postgres": 66.7, "rabbitmq": 91.2, "valkey": 11.9, "minio": 301.4, "loki": 87.8, "prometheus": 277.5}, "telemetry_delta": {"sdk_span_decisions_sample": 5553.0, "sdk_span_decisions_drop_ratio": 51184.0, "sdk_spans_exported_ok": 5553.0, "sdk_spans_exported_failed": null, "col_spans_in": 5553.0, "col_logs_in": 345.0, "col_points_in": 1290.0, "col_spans_out": 5553.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2546103.0, "tempo_spans_in": 5553.0, "tempo_bytes_in": 2070193.0, "loki_bytes_in": 137005.0, "loki_lines_in": 345.0, "prom_samples_in": 35514.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 530, "slow": 7, "error": 6, "pipeline_consumer": 6}, "window": [1794232280.3282502, 1794232370.689455], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "tail", "rep": 1, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232449.187008, "finished_at": 1794232539.181068, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5454, "successful_rps": 60.6, "status_counts": {"201": 4070, "202": 90, "200": 1294, "500": 36}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.29, "p95": 14.45, "p99": 42.01, "max": 106.41}, "faults_sent": {"slow": 56, "error": 36}, "by_kind": {"post_message": {"n": 4070, "p50_ms": 7.89, "p99_ms": 42.6}, "get_history": {"n": 1330, "p50_ms": 9.98, "p99_ms": 41.92}, "upload": {"n": 90, "p50_ms": 19.19, "p99_ms": 77.61}}}, "cpu_pct_avg": {"api-1": 10.92, "api-2": 11.23, "worker": 15.35, "otel-collector": 0.54, "tempo": 0.5, "nginx": 1.18, "grafana": 0.83, "postgres": 4.47, "rabbitmq": 6.56, "valkey": 0.47, "minio": 1.45, "loki": 0.52, "prometheus": 0.34}, "mem_mib_max": {"api-1": 137.4, "api-2": 116.8, "worker": 143.7, "otel-collector": 82.1, "tempo": 504.9, "nginx": 5.2, "grafana": 135.9, "postgres": 70.0, "rabbitmq": 157.4, "valkey": 11.9, "minio": 302.9, "loki": 90.2, "prometheus": 277.9}, "telemetry_delta": {"sdk_span_decisions_sample": 54799.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 54799.0, "sdk_spans_exported_failed": null, "col_spans_in": 56699.0, "col_logs_in": 326.0, "col_points_in": 1224.0, "col_spans_out": 10097.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21397727.0, "tempo_spans_in": 9711.0, "tempo_bytes_in": 3949068.0, "loki_bytes_in": 126917.0, "loki_lines_in": 326.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 5090.0, "baseline-10pct/true": 620.0, "keep-async-pipeline/false": 5619.0, "keep-async-pipeline/true": 91.0, "keep-errors/false": 5673.0, "keep-errors/true": 37.0, "keep-slow/false": 5649.0, "keep-slow/true": 61.0}}, "traces_in_tempo": {"server_api": 758, "slow": 56, "error": 36, "pipeline_consumer": 90}, "window": [1794232449.186968, 1794232539.195683], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "off", "rep": 2, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232618.428749, "finished_at": 1794232708.432197, "generator_backlog_at_end": 1, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"200": 1202, "202": 90, "201": 4147, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.33, "p95": 12.21, "p99": 32.69, "max": 68.21}, "faults_sent": {"slow": 56, "error": 51}, "by_kind": {"post_message": {"n": 4147, "p50_ms": 8.1, "p99_ms": 33.23}, "get_history": {"n": 1253, "p50_ms": 9.5, "p99_ms": 27.42}, "upload": {"n": 90, "p50_ms": 20.59, "p99_ms": 28.39}}}, "cpu_pct_avg": {"api-1": 8.49, "api-2": 8.47, "worker": 14.85, "otel-collector": 0.06, "tempo": 0.82, "nginx": 1.25, "grafana": 0.93, "postgres": 4.84, "rabbitmq": 6.54, "valkey": 0.49, "minio": 2.04, "loki": 0.54, "prometheus": 0.3}, "mem_mib_max": {"api-1": 93.7, "api-2": 93.3, "worker": 134.7, "otel-collector": 35.6, "tempo": 423.4, "nginx": 5.2, "grafana": 135.9, "postgres": 74.4, "rabbitmq": 154.2, "valkey": 11.9, "minio": 303.4, "loki": 89.5, "prometheus": 276.7}, "telemetry_delta": {"sdk_span_decisions_sample": -133.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -133.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794232618.428705, 1794232708.44837], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "on100", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232786.969322, "finished_at": 1794232876.97309, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5439, "successful_rps": 60.43, "status_counts": {"202": 90, "201": 4040, "200": 1309, "500": 51}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.09, "p95": 14.67, "p99": 43.98, "max": 89.3}, "faults_sent": {"slow": 57, "error": 51}, "by_kind": {"post_message": {"n": 4040, "p50_ms": 7.49, "p99_ms": 43.73}, "get_history": {"n": 1360, "p50_ms": 10.06, "p99_ms": 45.13}, "upload": {"n": 90, "p50_ms": 20.2, "p99_ms": 53.57}}}, "cpu_pct_avg": {"worker": 15.89, "api-1": 11.4, "api-2": 11.46, "otel-collector": 0.56, "tempo": 1.03, "nginx": 1.22, "grafana": 0.93, "postgres": 4.6, "rabbitmq": 6.71, "valkey": 0.45, "minio": 1.71, "loki": 0.58, "prometheus": 0.33}, "mem_mib_max": {"worker": 155.8, "api-1": 121.5, "api-2": 107.1, "otel-collector": 74.9, "tempo": 715.3, "nginx": 5.5, "grafana": 136.0, "postgres": 84.5, "rabbitmq": 154.9, "valkey": 12.1, "minio": 304.7, "loki": 91.0, "prometheus": 281.9}, "telemetry_delta": {"sdk_span_decisions_sample": 55086.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55086.0, "sdk_spans_exported_failed": null, "col_spans_in": 56986.0, "col_logs_in": 341.0, "col_points_in": 1224.0, "col_spans_out": 56986.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21513949.0, "tempo_spans_in": 56986.0, "tempo_bytes_in": 21018520.0, "loki_bytes_in": 136272.0, "loki_lines_in": 341.0, "prom_samples_in": 34616.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 57, "error": 51, "pipeline_consumer": 90}, "window": [1794232786.969286, 1794232876.987838], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "head10", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794232959.123384, "finished_at": 1794233049.151171, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5442, "successful_rps": 60.47, "status_counts": {"200": 1281, "201": 4071, "202": 90, "500": 48}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.6, "p95": 10.27, "p99": 36.75, "max": 114.16}, "faults_sent": {"slow": 62, "error": 48}, "by_kind": {"post_message": {"n": 4071, "p50_ms": 6.31, "p99_ms": 33.63}, "get_history": {"n": 1329, "p50_ms": 8.13, "p99_ms": 38.58}, "upload": {"n": 90, "p50_ms": 16.04, "p99_ms": 74.32}}}, "cpu_pct_avg": {"api-1": 9.85, "api-2": 9.72, "worker": 15.02, "otel-collector": 0.23, "tempo": 0.44, "nginx": 1.23, "grafana": 0.91, "postgres": 4.53, "rabbitmq": 6.55, "valkey": 0.51, "minio": 1.56, "loki": 0.55, "prometheus": 0.34}, "mem_mib_max": {"api-1": 99.3, "api-2": 101.9, "worker": 156.9, "otel-collector": 75.6, "tempo": 408.7, "nginx": 5.2, "grafana": 136.0, "postgres": 90.7, "rabbitmq": 91.4, "valkey": 12.4, "minio": 305.8, "loki": 92.2, "prometheus": 282.7}, "telemetry_delta": {"sdk_span_decisions_sample": 5559.0, "sdk_span_decisions_drop_ratio": 50970.0, "sdk_spans_exported_ok": 5559.0, "sdk_spans_exported_failed": null, "col_spans_in": 5559.0, "col_logs_in": 338.0, "col_points_in": 1296.0, "col_spans_out": 5559.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2555586.0, "tempo_spans_in": 5559.0, "tempo_bytes_in": 2082660.0, "loki_bytes_in": 132915.0, "loki_lines_in": 338.0, "prom_samples_in": 35520.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 554, "slow": 3, "error": 4, "pipeline_consumer": 9}, "window": [1794232959.123342, 1794233049.1668348], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "tail", "rep": 2, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233127.526659, "finished_at": 1794233217.5204532, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5424, "successful_rps": 60.27, "status_counts": {"201": 3988, "202": 90, "200": 1346, "500": 66}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.71, "p95": 11.88, "p99": 43.18, "max": 135.99}, "faults_sent": {"slow": 70, "error": 66}, "by_kind": {"post_message": {"n": 3988, "p50_ms": 6.4, "p99_ms": 42.94}, "get_history": {"n": 1412, "p50_ms": 8.66, "p99_ms": 59.68}, "upload": {"n": 90, "p50_ms": 17.45, "p99_ms": 108.82}}}, "cpu_pct_avg": {"api-1": 11.36, "api-2": 11.19, "worker": 15.2, "otel-collector": 0.53, "tempo": 0.63, "nginx": 1.22, "grafana": 0.84, "postgres": 4.41, "rabbitmq": 6.66, "valkey": 0.47, "minio": 1.52, "loki": 0.53, "prometheus": 0.32}, "mem_mib_max": {"api-1": 102.4, "api-2": 103.0, "worker": 148.8, "otel-collector": 77.9, "tempo": 486.8, "nginx": 5.4, "grafana": 135.6, "postgres": 96.5, "rabbitmq": 151.3, "valkey": 12.1, "minio": 307.6, "loki": 93.8, "prometheus": 282.2}, "telemetry_delta": {"sdk_span_decisions_sample": 55543.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55543.0, "sdk_spans_exported_failed": null, "col_spans_in": 57443.0, "col_logs_in": 356.0, "col_points_in": 1326.0, "col_spans_out": 10531.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21716724.0, "tempo_spans_in": 9969.0, "tempo_bytes_in": 4062445.0, "loki_bytes_in": 145752.0, "loki_lines_in": 356.0, "prom_samples_in": 41766.0, "tail_policies": {"baseline-10pct/false": 5284.0, "baseline-10pct/true": 612.0, "keep-async-pipeline/false": 5804.0, "keep-async-pipeline/true": 92.0, "keep-errors/false": 5826.0, "keep-errors/true": 70.0, "keep-slow/false": 5824.0, "keep-slow/true": 72.0}}, "traces_in_tempo": {"server_api": 774, "slow": 70, "error": 66, "pipeline_consumer": 90}, "window": [1794233127.526627, 1794233217.535255], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "off", "rep": 3, "env": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233296.514442, "finished_at": 1794233386.5104158, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5440, "successful_rps": 60.44, "status_counts": {"201": 4051, "202": 90, "200": 1299, "500": 50}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.56, "p95": 10.45, "p99": 33.86, "max": 79.09}, "faults_sent": {"slow": 52, "error": 50}, "by_kind": {"post_message": {"n": 4051, "p50_ms": 6.23, "p99_ms": 33.4}, "get_history": {"n": 1349, "p50_ms": 7.95, "p99_ms": 50.36}, "upload": {"n": 90, "p50_ms": 18.98, "p99_ms": 37.17}}}, "cpu_pct_avg": {"api-1": 9.13, "api-2": 9.18, "worker": 15.32, "otel-collector": 0.08, "tempo": 0.86, "nginx": 1.34, "grafana": 0.98, "postgres": 5.15, "rabbitmq": 6.66, "valkey": 0.57, "minio": 1.66, "loki": 0.6, "prometheus": 0.36}, "mem_mib_max": {"api-1": 105.8, "api-2": 124.5, "worker": 144.7, "otel-collector": 35.6, "tempo": 695.5, "nginx": 5.2, "grafana": 136.0, "postgres": 94.7, "rabbitmq": 157.3, "valkey": 12.6, "minio": 309.7, "loki": 95.0, "prometheus": 289.2}, "telemetry_delta": {"sdk_span_decisions_sample": -209.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": -209.0, "sdk_spans_exported_failed": null, "col_spans_in": null, "col_logs_in": null, "col_points_in": null, "col_spans_out": null, "col_spans_refused": null, "col_spans_dropped": 0.0, "col_otlp_bytes_in": null, "tempo_spans_in": 0.0, "tempo_bytes_in": 0.0, "loki_bytes_in": 0.0, "loki_lines_in": 0.0, "prom_samples_in": 23378.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 0, "slow": 0, "error": 0, "pipeline_consumer": 0}, "window": [1794233296.514399, 1794233386.524056], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "on100", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233465.4906292, "finished_at": 1794233555.486665, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5445, "successful_rps": 60.5, "status_counts": {"201": 4032, "202": 90, "200": 1323, "500": 45}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.63, "p95": 11.84, "p99": 43.42, "max": 76.29}, "faults_sent": {"slow": 50, "error": 45}, "by_kind": {"post_message": {"n": 4032, "p50_ms": 6.34, "p99_ms": 42.73}, "get_history": {"n": 1368, "p50_ms": 8.43, "p99_ms": 43.79}, "upload": {"n": 90, "p50_ms": 18.37, "p99_ms": 75.49}}}, "cpu_pct_avg": {"api-1": 11.12, "api-2": 11.21, "worker": 15.33, "otel-collector": 0.54, "tempo": 1.29, "nginx": 1.19, "grafana": 0.89, "postgres": 4.63, "rabbitmq": 6.56, "valkey": 0.49, "minio": 1.7, "loki": 0.57, "prometheus": 0.38}, "mem_mib_max": {"api-1": 102.6, "api-2": 107.7, "worker": 143.0, "otel-collector": 73.6, "tempo": 575.9, "nginx": 5.2, "grafana": 135.5, "postgres": 99.4, "rabbitmq": 92.2, "valkey": 12.3, "minio": 311.5, "loki": 94.7, "prometheus": 286.5}, "telemetry_delta": {"sdk_span_decisions_sample": 55247.0, "sdk_span_decisions_drop_ratio": null, "sdk_spans_exported_ok": 55247.0, "sdk_spans_exported_failed": null, "col_spans_in": 57147.0, "col_logs_in": 335.0, "col_points_in": 1224.0, "col_spans_out": 57147.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21561062.0, "tempo_spans_in": 57147.0, "tempo_bytes_in": 21069123.0, "loki_bytes_in": 132480.0, "loki_lines_in": 335.0, "prom_samples_in": 35456.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 5510, "slow": 50, "error": 45, "pipeline_consumer": 90}, "window": [1794233465.490568, 1794233555.500811], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "head10", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233634.709635, "finished_at": 1794233725.1747258, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5427, "successful_rps": 60.3, "status_counts": {"201": 4006, "202": 90, "200": 1331, "500": 63}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 6.57, "p95": 10.44, "p99": 36.56, "max": 264.95}, "faults_sent": {"slow": 68, "error": 63}, "by_kind": {"post_message": {"n": 4006, "p50_ms": 6.25, "p99_ms": 29.64}, "get_history": {"n": 1394, "p50_ms": 8.14, "p99_ms": 44.81}, "upload": {"n": 90, "p50_ms": 18.47, "p99_ms": 63.61}}}, "cpu_pct_avg": {"api-1": 10.01, "api-2": 10.04, "worker": 15.08, "otel-collector": 0.23, "tempo": 0.43, "nginx": 1.25, "grafana": 0.88, "postgres": 4.63, "rabbitmq": 6.55, "valkey": 0.48, "minio": 1.74, "loki": 0.58, "prometheus": 0.34}, "mem_mib_max": {"api-1": 101.0, "api-2": 99.4, "worker": 148.2, "otel-collector": 69.7, "tempo": 359.2, "nginx": 5.1, "grafana": 136.2, "postgres": 108.8, "rabbitmq": 156.7, "valkey": 12.1, "minio": 314.6, "loki": 122.6, "prometheus": 284.1}, "telemetry_delta": {"sdk_span_decisions_sample": 5568.0, "sdk_span_decisions_drop_ratio": 51652.0, "sdk_spans_exported_ok": 5568.0, "sdk_spans_exported_failed": null, "col_spans_in": 5568.0, "col_logs_in": 353.0, "col_points_in": 1287.0, "col_spans_out": 5568.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 2566444.0, "tempo_spans_in": 5568.0, "tempo_bytes_in": 2085852.0, "loki_bytes_in": 142091.0, "loki_lines_in": 353.0, "prom_samples_in": 39528.0, "tail_policies": {}}, "traces_in_tempo": {"server_api": 546, "slow": 8, "error": 5, "pipeline_consumer": 10}, "window": [1794233634.709598, 1794233725.183886], "stats_samples": 14, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
{"config": "tail", "rep": 3, "env": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"}, "load": {"profile": {"rate_rps": 60, "duration_s": 90, "upload_rate_rps": 1, "slow_pct": 1, "error_pct": 1, "threads": 96, "mix": "75% POST message / 25% GET history(limit=20); uploads separate"}, "started_at": 1794233805.530024, "finished_at": 1794233895.525802, "generator_backlog_at_end": 0, "requests": 5490, "ok": 5428, "successful_rps": 60.31, "status_counts": {"201": 4009, "202": 90, "200": 1329, "500": 62}, "server_errors_unexpected": 0, "latency_ms_plain_ok": {"p50": 8.69, "p95": 14.69, "p99": 48.23, "max": 153.66}, "faults_sent": {"slow": 54, "error": 62}, "by_kind": {"post_message": {"n": 4009, "p50_ms": 8.35, "p99_ms": 45.87}, "get_history": {"n": 1391, "p50_ms": 10.45, "p99_ms": 51.17}, "upload": {"n": 90, "p50_ms": 20.58, "p99_ms": 84.12}}}, "cpu_pct_avg": {"worker": 15.24, "api-1": 10.99, "api-2": 11.11, "otel-collector": 0.52, "tempo": 0.8, "nginx": 1.16, "grafana": 0.88, "postgres": 4.49, "rabbitmq": 6.64, "valkey": 0.46, "minio": 1.69, "loki": 0.5, "prometheus": 0.32}, "mem_mib_max": {"worker": 143.4, "api-1": 106.7, "api-2": 102.9, "otel-collector": 83.0, "tempo": 487.7, "nginx": 5.1, "grafana": 135.8, "postgres": 114.2, "rabbitmq": 157.5, "valkey": 11.9, "minio": 316.2, "loki": 114.4, "prometheus": 284.0}, "telemetry_delta": {"sdk_span_decisions_sample": 55277.0, "sdk_span_decisions_drop_ratio": 0.0, "sdk_spans_exported_ok": 55277.0, "sdk_spans_exported_failed": null, "col_spans_in": 57177.0, "col_logs_in": 352.0, "col_points_in": 1224.0, "col_spans_out": 9512.0, "col_spans_refused": 0.0, "col_spans_dropped": 0.0, "col_otlp_bytes_in": 21589232.0, "tempo_spans_in": 9559.0, "tempo_bytes_in": 3899582.0, "loki_bytes_in": 143224.0, "loki_lines_in": 352.0, "prom_samples_in": 36752.0, "tail_policies": {"baseline-10pct/false": 4962.0, "baseline-10pct/true": 548.0, "keep-async-pipeline/false": 5420.0, "keep-async-pipeline/true": 90.0, "keep-errors/false": 5448.0, "keep-errors/true": 62.0, "keep-slow/false": 5456.0, "keep-slow/true": 54.0}}, "traces_in_tempo": {"server_api": 737, "slow": 54, "error": 62, "pipeline_consumer": 90}, "window": [1794233805.529983, 1794233895.540648], "stats_samples": 13, "hardware": {"docker": "29.7.2", "vm_cpus": 10, "vm_mem_gib": 7.7}}
+31
View File
@@ -0,0 +1,31 @@
Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.
| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |
|---|---|---|---|---|---|---|---|---|
| off | 3 | 60.4 | 0 | 7.1 (6.4…8.3) | 11.0 (10.3…12.2) | 32.7 (31.7…33.9) | 17.8 (17.0…18.3) | 15.2 (14.8…15.3) |
| on100 | 3 | 60.5 | 0 | 7.8 (6.6…8.6) | 14.3 (11.8…16.3) | 47.2 (43.4…54.4) | 22.7 (22.3…22.9) | 15.6 (15.3…15.9) |
| head10 | 3 | 60.4 | 0 | 7.2 (6.6…8.3) | 11.2 (10.3…13.0) | 37.5 (36.6…39.2) | 19.7 (19.6…20.0) | 15.0 (14.9…15.1) |
| tail | 3 | 60.4 | 0 | 7.9 (6.7…8.7) | 13.7 (11.9…14.7) | 44.5 (42.0…48.2) | 22.3 (22.1…22.5) | 15.3 (15.2…15.3) |
| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |
|---|---|---|---|---|---|---|---|
| off | 0.07 (0.06…0.08) | 0.70 (0.43…0.86) | 0.57 (0.54…0.60) | 0.34 (0.30…0.36) | 98 | 36 | 470 |
| on100 | 0.57 (0.54…0.60) | 1.13 (1.03…1.29) | 0.57 (0.55…0.58) | 0.36 (0.33…0.38) | 113 | 73 | 571 |
| head10 | 0.23 (0.23…0.24) | 0.41 (0.35…0.44) | 0.54 (0.50…0.58) | 0.34 (0.33…0.34) | 105 | 71 | 423 |
| tail | 0.53 (0.52…0.54) | 0.64 (0.50…0.80) | 0.52 (0.50…0.53) | 0.33 (0.32…0.34) | 116 | 81 | 493 |
| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |
|---|---|---|---|---|---|---|
| off | — | 0 | 0 | — | 0 | 0 |
| on100 | 57059 | 57059 | 0 | 21030 | 20547 | 132 |
| head10 | 5560 | 5560 | 0 | 2496 | 2031 | 134 |
| tail | 57106 | 9746 | 0 | 21062 | 3877 | 135 |
Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.
| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |
|---|---|---|---|---|---|
| off | 0 / 16470 | 0 / 164 | 0 / 163 | 0 / 270 | — / 163 |
| on100 | 16530 / 16470 | 167 / 167 | 146 / 146 | 270 / 270 | 146 / 146 |
| head10 | 1630 / 16470 | 18 / 198 | 15 / 166 | 25 / 270 | 166 / 166 |
| tail | 2269 / 16470 | 180 / 180 | 164 / 164 | 270 / 270 | 164 / 164 |
+20
View File
@@ -0,0 +1,20 @@
{
"alert_pending_s": 60.01730489730835,
"alert_firing_s": 340,
"saturation_at_firing": 1.0,
"queued_at_firing": 194.0,
"oldest_age_s": 63,
"jobs_per_s": 3.16,
"job_p95_s": 0.98,
"alerts_at_firing": {
"MessengerApiHighErrorRate": "firing",
"MessengerQueueLagGrowing": "firing",
"MessengerWorkerSaturated": "firing"
},
"resolved_after_restore_s": 17,
"drained_after_restore_s": 22,
"uploads": {
"202": 1255,
"503": 108
}
}
+10
View File
@@ -0,0 +1,10 @@
17:19:44 lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas
17:19:51 uploads: 4/s open-loop (sample.png)
17:20:51 fresh capacity=2 slots (stale series of recreated containers are excluded)
17:20:51 MessengerWorkerSaturated pending (+60 s)
17:25:31 MessengerWorkerSaturated firing (+340 s)
17:25:31 at firing: saturation=1.00, queued=194, oldest age=63 s, succeeded=3.16 jobs/s, job p95=0.98 s, alerts={'MessengerApiHighErrorRate': 'firing', 'MessengerQueueLagGrowing': 'firing', 'MessengerWorkerSaturated': 'firing'}
17:25:31 restore default worker configuration
17:25:49 MessengerWorkerSaturated resolved (+17 s)
17:25:54 backlog drained: True (+22 s after restore)
PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,44 @@
# Grafana 11.5.2 datasources (fixed uids are referenced by the dashboard JSON). Grafana stores no signals itself.
# "$$" escapes "$" from Grafana's env-variable interpolation in provisioning files.
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 15s
prometheusType: Prometheus
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
maxLines: 1000
derivedFields:
# trace_id is OTLP structured metadata -> link "Open trace" to Tempo.
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
urlDisplayLabel: Open trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-5m"
spanEndTimeShift: "5m"
filterByTraceID: true
filterBySpanID: false
customQuery: true
query: '{service_name=~"messenger-.*"} | trace_id="$${__trace.traceId}"'
nodeGraph:
enabled: true
+39
View File
@@ -0,0 +1,39 @@
# Loki 3.4.2 single binary: log store. Logs arrive by OTLP (http://loki:3100/otlp) from the Collector.
# service.name -> stream label service_name; log attributes (event, code.*) and trace_id/span_id -> structured metadata.
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2024-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
volume_enabled: true
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*", "arn:aws:s3:::messenger-attachments/staging/*"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API.
set -eu
BUCKET="${S3_BUCKET:-messenger-attachments}"
ROOT_USER="$(cat /run/secrets/minio_root_user)"
ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
APP_USER="$(cat /run/secrets/s3_access_key_id)"
APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)"
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
echo "waiting for minio"; sleep 2
done
mc mb --ignore-existing "local/$BUCKET"
mc anonymous set none "local/$BUCKET"
mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \
|| mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true
mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true
mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true
echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*"
+63
View File
@@ -0,0 +1,63 @@
#!/bin/sh
set -u
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
INTERVAL="${HEALTH_INTERVAL:-1}"
FALL="${HEALTH_FALL:-2}"
RISE="${HEALTH_RISE:-2}"
OUT=/etc/nginx/upstream/servers.conf
mkdir -p "$(dirname "$OUT")"
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
render() {
any_up=0
for s in $SERVERS; do
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
done
for s in $SERVERS; do
mark=""
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
mark=" down"
fi
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
done
}
for s in $SERVERS; do
v=$(var "$s")
eval "up_$v=1 ok_$v=0 bad_$v=0"
done
render > "$OUT"
watch() {
pending=0
while sleep "$INTERVAL"; do
changed=0
for s in $SERVERS; do
v=$(var "$s")
if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
fi
else
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
fi
fi
done
if [ "$changed" = 1 ]; then
render > "$OUT"
pending=1
fi
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
pending=0
fi
done
}
watch &
exec nginx -g 'daemon off;'
+75
View File
@@ -0,0 +1,75 @@
worker_processes 2;
events {
worker_connections 256;
}
http {
resolver 127.0.0.11 valid=5s ipv6=off;
upstream api {
zone api 64k;
include /etc/nginx/upstream/servers.conf;
keepalive 32;
keepalive_timeout 4s;
}
log_format main '$remote_addr - $upstream_addr [$time_local] '
'"$request" $status $body_bytes_sent '
'"$upstream_response_time" "$request_time"';
access_log /var/log/nginx/access.log main;
server {
listen 8080;
client_max_body_size 64k;
error_page 413 @payload_too_large;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 5s;
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_pass http://api;
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
}
location ~ ^/api/v1/chats/[^/]+/attachments$ {
client_max_body_size 10m;
proxy_request_buffering off;
proxy_pass http://api;
proxy_next_upstream off;
}
location @payload_too_large {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header X-Instance-Id lb always;
add_header Connection close always;
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
}
location /health/ {
proxy_pass http://api;
proxy_next_upstream error timeout;
}
location ~ ^/api/v1/attachments/ {
proxy_max_temp_file_size 0;
proxy_pass http://api;
proxy_next_upstream error timeout;
}
location / {
proxy_pass http://api;
proxy_next_upstream error timeout http_502 http_503;
}
}
}
+88
View File
@@ -0,0 +1,88 @@
# OpenTelemetry Collector (contrib 0.120.0): receiver/processor/exporter only, it stores nothing long-term.
# API, worker --OTLP/HTTP:4318--> collector --> Prometheus (metrics, OTLP), Loki (logs, OTLP), Tempo (traces, OTLP/gRPC)
# Ports are reachable only inside the compose network; nothing here is published to the host.
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 4194304 # 4 MiB per OTLP request
processors:
# First in every pipeline: refuse new data (receivers answer 503 -> SDK retries/drops) before the container OOMs.
# Container limit is 256 MiB; soft limit = limit_mib - spike_limit_mib = 140 MiB.
memory_limiter:
check_interval: 1s
limit_mib: 180
spike_limit_mib: 40
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
# Defense in depth: even if an instrumentation adds them, these attributes never leave the collector.
# url.query would contain ?filename=...; headers could contain Authorization.
attributes/scrub:
actions:
- {key: url.query, action: delete}
- {key: url.full, action: delete}
- {key: http.target, action: delete}
- {key: http.url, action: delete}
- {key: http.request.header.authorization, action: delete}
- {key: http.request.header.cookie, action: delete}
- {key: db.query.parameters, action: delete}
exporters:
# Every exporter: bounded in-memory queue (batches) + bounded retry. Backend down => queue fills, then new
# batches are dropped and counted (otelcol_exporter_enqueue_failed_*); RAM stays bounded, apps are unaffected.
otlphttp/prometheus:
endpoint: http://prometheus:9090/api/v1/otlp
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
otlphttp/loki:
endpoint: http://loki:3100/otlp
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
otlp/tempo:
endpoint: tempo:4317
tls: {insecure: true}
compression: gzip
timeout: 5s
sending_queue: {enabled: true, num_consumers: 2, queue_size: 100}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 120s}
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlphttp/loki]
+5
View File
@@ -0,0 +1,5 @@
# Default profile: no extra processing. All spans that the SDK head-sampled are exported.
# Selected with COLLECTOR_PROFILE=default (see compose.yaml); merged over collector.yaml.
processors:
batch:
timeout: 2s
+31
View File
@@ -0,0 +1,31 @@
# Track 1 "balanced" profile (COLLECTOR_PROFILE=tail): SDK keeps 100% head sampling, the collector decides per
# whole trace after decision_wait. Keeps: every trace with an error, every trace slower than 500 ms, every trace of
# the attachment pipeline (rare, async, valuable), and 10% of the remaining ordinary traffic.
# Metrics are computed in the SDK BEFORE any sampling, so RPS/error rate/percentiles are unaffected.
processors:
tail_sampling:
decision_wait: 10s
num_traces: 20000
expected_new_traces_per_sec: 200
decision_cache:
sampled_cache_size: 50000
policies:
- name: keep-errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: keep-slow
type: latency
latency: {threshold_ms: 500}
- name: keep-async-pipeline
type: string_attribute
string_attribute: {key: messaging.system, values: [rabbitmq]}
- name: baseline-10pct
type: probabilistic
probabilistic: {sampling_percentage: 10}
service:
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, tail_sampling, batch]
exporters: [otlp/tempo]
+31
View File
@@ -0,0 +1,31 @@
# Prometheus v3.2.1: metrics store. Application metrics arrive by OTLP push from the Collector
# (--web.enable-otlp-receiver, /api/v1/otlp); infrastructure self-metrics are scraped.
global:
scrape_interval: 15s
evaluation_interval: 15s
otlp:
# job=service.name, instance=service.instance.id are set by Prometheus; these become extra labels.
promote_resource_attributes:
- service.name
- service.instance.id
- service.version
- deployment.environment.name
storage:
tsdb:
# OTLP pushes from several SDKs/batches may arrive slightly out of order.
out_of_order_time_window: 10m
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: otel-collector
static_configs: [{targets: ["otel-collector:8888"]}]
- job_name: tempo
static_configs: [{targets: ["tempo:3200"]}]
- job_name: loki
static_configs: [{targets: ["loki:3100"]}]
- job_name: prometheus
static_configs: [{targets: ["localhost:9090"]}]
+111
View File
@@ -0,0 +1,111 @@
# Recording rules (SLI/SLO) and alert rules. Evaluated by Prometheus every 15 s; state (pending/firing/inactive)
# is visible in Prometheus /alerts, the API /api/v1/alerts and in Grafana (alert list panel). No external delivery.
# Runbooks: docs/runbooks.md (anchors in runbook_url).
#
# Label mapping (OTLP -> Prometheus 3): service.name -> service_name, service.instance.id -> instance,
# http.route -> http_route, http.response.status_code -> http_response_status_code, job.type -> job_type.
# HTTP SLI denominator: every business request of messenger-api (health probes are not instrumented).
# Numerator of errors: 5xx only. 4xx (wrong password, 404 of a foreign chat, 409) are client outcomes, not failures.
groups:
- name: messenger-sli
interval: 15s
rules:
- record: messenger:http_requests:rate2m
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api"}[2m]))
- record: messenger:http_errors_5xx:rate2m
expr: sum(rate(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[2m])) or vector(0)
- record: messenger:http_error_ratio:rate2m
expr: messenger:http_errors_5xx:rate2m / messenger:http_requests:rate2m
- record: messenger:http_latency_p95:rate5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name="messenger-api"}[5m])))
# SLO window: rolling 30 min (lab-sized; production would use 28-30 days with the same formulas).
# Availability SLO: 99% of business requests without 5xx.
- record: messenger:slo_availability:ratio30m
expr: |
1 - (
(sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code=~"5.."}[30m])) or vector(0))
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
)
# Latency SLO: 95% of business requests complete in <= 500 ms (bucket le="0.5").
- record: messenger:slo_latency:ratio30m
expr: |
sum(increase(http_server_request_duration_seconds_bucket{service_name="messenger-api",le="0.5"}[30m]))
/ sum(increase(http_server_request_duration_seconds_count{service_name="messenger-api"}[30m]))
- record: messenger:worker_saturation:ratio
# OTLP-pushed series have no staleness markers: a recreated worker would stay in the denominator for the
# 5 min lookback. Only samples fresher than 30 s (3 export intervals) are counted.
expr: |
sum(messenger_worker_busy and (time() - timestamp(messenger_worker_busy) < 30))
/ sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))
- name: messenger-alerts
rules:
- alert: MessengerApiHighErrorRate
expr: messenger:http_error_ratio:rate2m > 0.05 and messenger:http_requests:rate2m > 0.2
for: 1m
labels:
severity: page
annotations:
summary: "API: доля 5xx {{ $value | humanizePercentage }} за 2 мин (порог 5%)"
description: "Больше 5% бизнес-запросов messenger-api завершаются 5xx (окно 2m, держится 1m). Расходуется бюджет SLO доступности 99%."
runbook_url: "docs/runbooks.md#messengerapihigherrorrate"
- alert: MessengerQueueLagGrowing
expr: max(messenger_queue_oldest_age_seconds{queue="attachments.process"}) > 60
for: 1m
labels:
severity: page
annotations:
summary: "Очередь вложений: старейшее задание ждёт {{ $value | humanizeDuration }}"
description: "Принятое задание не завершено дольше 60 с (порог), условие держится 1m: worker остановлен, перегружен или не может писать в S3/DB."
runbook_url: "docs/runbooks.md#messengerqueuelaggrowing"
- alert: MessengerApiLatencyHigh
expr: messenger:http_latency_p95:rate5m > 1
for: 2m
labels:
severity: ticket
annotations:
summary: "API p95 = {{ $value | humanizeDuration }} (порог 1 s, окно 5m)"
description: "p95 из histogram buckets всех реплик выше 1 s."
runbook_url: "docs/runbooks.md#messengerapilatencyhigh"
- alert: MessengerDbPoolWaiting
expr: max(db_client_connection_pending_requests) > 0
for: 1m
labels:
severity: ticket
annotations:
summary: "Запросы ждут соединение из пула БД ({{ $value }} в очереди)"
description: "Пул psycopg исчерпан дольше минуты: БД медленная/блокирована или пул мал."
runbook_url: "docs/runbooks.md#messengerdbpoolwaiting"
- alert: MessengerWorkerSaturated
expr: messenger:worker_saturation:ratio > 0.9
for: 5m
labels:
severity: ticket
annotations:
summary: "Worker занят на {{ $value | humanizePercentage }} ёмкости 5 минут"
runbook_url: "docs/runbooks.md#messengerworkersaturated"
- alert: TelemetryExportFailing
# Each term has "or vector(0)": the *_failed_* series do not exist until the first failure, and a sum with an
# empty operand is empty (the alert would never fire). Early signal: exporter queue filling up (backend
# down, retries in progress); late signal: data dropped (queue full) or retries exhausted.
expr: |
(
(sum(rate(otelcol_exporter_send_failed_spans_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_send_failed_log_records_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_enqueue_failed_spans_total[2m])) or vector(0))
+ (sum(rate(otelcol_exporter_enqueue_failed_log_records_total[2m])) or vector(0))
) > 0
or
max(max by (exporter) (otelcol_exporter_queue_size) / max by (exporter) (otelcol_exporter_queue_capacity)) > 0.2
for: 1m
labels:
severity: ticket
annotations:
summary: "Collector не может доставить telemetry в backend"
description: "Очередь exporter заполнена >20% или данные отбрасываются: Tempo/Loki/Prometheus недоступен. Бизнес-API не затронут, RAM collector ограничена очередью и memory_limiter."
runbook_url: "docs/runbooks.md#telemetryexportfailing"
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
# Credentials come from a Docker secret; RabbitMQ's config cannot read files, so render a conf.d fragment at start.
set -eu
PASS="$(cat /run/secrets/rabbitmq_password)"
mkdir -p /etc/rabbitmq/conf.d
cat > /etc/rabbitmq/conf.d/20-credentials.conf <<CONF
default_user = messenger
default_pass = ${PASS}
default_vhost = messenger
default_permissions.configure = .*
default_permissions.read = .*
default_permissions.write = .*
CONF
exec docker-entrypoint.sh rabbitmq-server
+10
View File
@@ -0,0 +1,10 @@
# Durable broker for the attachment pipeline (see REPORT.md).
loopback_users.guest = true
listeners.tcp.default = 5672
# A delivery not acked for 10 min is returned to the queue (worker hang guard on the broker side;
# the application lease in PostgreSQL detects a hung job much earlier).
consumer_timeout = 600000
# Refuse publishes before the disk fills up (publisher sees the block and the outbox keeps the jobs).
disk_free_limit.absolute = 200MB
vm_memory_high_watermark.relative = 0.6
log.console.level = warning
+28
View File
@@ -0,0 +1,28 @@
# Tempo 2.7.1 single binary: trace store (local blocks on a volume). Receives OTLP/gRPC from the Collector.
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
query_frontend:
search:
max_result_limit: 10000
usage_report:
reporting_enabled: false
+46
View File
@@ -0,0 +1,46 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name bytea NOT NULL
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
created_at timestamptz NOT NULL
);
CREATE INDEX chats_created_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
text bytea NOT NULL
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
created_at timestamptz NOT NULL
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
key text NOT NULL
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
request_body_hash text NOT NULL,
response_json text NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+28
View File
@@ -0,0 +1,28 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
filename text NOT NULL
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL,
status text NOT NULL DEFAULT 'ready'
CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_size;
ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000);
@@ -0,0 +1,5 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed'));
ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT;
CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key);
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
+75
View File
@@ -0,0 +1,75 @@
-- lab5: upload -> staging -> job/outbox -> broker -> worker -> final prefix + variants.
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed'));
-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker.
ALTER TABLE attachments ADD COLUMN staging_key text;
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL;
-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0),
width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0),
height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0),
sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL,
PRIMARY KEY (attachment_id, pipeline_version, name)
);
-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry.
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
state text NOT NULL
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')),
attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1),
max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1),
next_attempt_at timestamptz NOT NULL,
lease_owner text,
lease_until timestamptz,
error_code text,
last_error text,
created_at timestamptz NOT NULL,
first_started_at timestamptz,
started_at timestamptz,
finished_at timestamptz,
updated_at timestamptz NOT NULL,
UNIQUE (attachment_id, pipeline_version)
);
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing';
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed';
CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing');
-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms.
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
payload jsonb NOT NULL,
available_at timestamptz NOT NULL,
published_at timestamptz,
publish_attempts integer NOT NULL DEFAULT 0,
last_error text,
created_at timestamptz NOT NULL
);
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL;
CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL;
-- Operator-visible history of every state change of a job (including explicit replay).
CREATE TABLE job_audit (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL,
action text NOT NULL,
actor text NOT NULL,
attempt integer NOT NULL,
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz NOT NULL
);
CREATE INDEX job_audit_job_idx ON job_audit (job_id, id);
+60
View File
@@ -0,0 +1,60 @@
aio-pika==9.5.5
aiormq==6.8.1
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
asgiref==3.13.0
boto3==1.35.81
botocore==1.35.81
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
Deprecated==3.0.1
exceptiongroup==1.2.2
fastapi==0.115.6
googleapis-common-protos==1.75.0
h11==0.16.0
httptools==0.9.0
idna==3.20
importlib_metadata==8.5.0
jmespath==1.0.1
multidict==6.1.0
opentelemetry-api==1.30.0
opentelemetry-exporter-otlp-proto-common==1.30.0
opentelemetry-exporter-otlp-proto-http==1.30.0
opentelemetry-instrumentation==0.51b0
opentelemetry-instrumentation-asgi==0.51b0
opentelemetry-instrumentation-botocore==0.51b0
opentelemetry-instrumentation-fastapi==0.51b0
opentelemetry-propagator-aws-xray==1.0.2
opentelemetry-proto==1.30.0
opentelemetry-sdk==1.30.0
opentelemetry-semantic-conventions==0.51b0
opentelemetry-util-http==0.51b0
packaging==26.3
pamqp==3.3.0
pillow==11.0.0
propcache==0.2.1
protobuf==5.29.6
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
pycparser==3.11
pydantic==2.14.1
pydantic_core==2.50.1
python-dateutil==2.9.0.post0
requests==2.34.2
s3transfer==0.10.4
six==1.17.0
starlette==0.41.3
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.3.0
uvicorn==0.34.0
uvloop==0.23.0
valkey==6.2.0
wrapt==1.17.3
yarl==1.18.3
zipp==4.1.2
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Generates infra/grafana/dashboards/messenger.json (provisioned into Grafana at startup).
The JSON is committed; this script only keeps it readable/editable. Run: python3 scripts/gen_dashboard.py
Every expression uses metric names that really exist in Prometheus (see REPORT.md, "Схема сигналов").
"""
import json
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "infra/grafana/dashboards/messenger.json"
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
API = 'service_name="messenger-api"'
HTTP = "http_server_request_duration_seconds"
_ids = iter(range(1, 1000))
def target(expr, legend="", ref="A", instant=False):
return {"datasource": PROM, "expr": expr, "legendFormat": legend, "refId": ref, "range": not instant,
"instant": instant}
def ts(title, targets, unit="short", x=0, y=0, w=12, h=8, desc="", stack=False, extra=None):
panel = {
"id": next(_ids), "type": "timeseries", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10,
"stacking": {"mode": "normal" if stack else "none"},
"showPoints": "never"}},
"overrides": []},
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
"tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [target(*t) if isinstance(t, tuple) else t for t in targets],
}
if extra:
panel["fieldConfig"]["defaults"].update(extra)
return panel
def stat(title, expr, unit, x, y, w=4, h=4, desc="", thresholds=None, decimals=None):
steps = thresholds or [{"color": "green", "value": None}]
panel = {
"id": next(_ids), "type": "stat", "title": title, "description": desc, "datasource": PROM,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"fieldConfig": {"defaults": {"unit": unit, "thresholds": {"mode": "absolute", "steps": steps}},
"overrides": []},
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"colorMode": "value", "graphMode": "area", "textMode": "auto"},
"targets": [target(expr, title)],
}
if decimals is not None:
panel["fieldConfig"]["defaults"]["decimals"] = decimals
return panel
def row(title, y):
return {"id": next(_ids), "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1},
"panels": []}
def q(p, expr_inner):
return "histogram_quantile(%s, %s)" % (p, expr_inner)
panels = []
y = 0
# ------------------------------------------------------------------ SLO and alerts
panels.append(row("SLO (окно 30 мин) и алерты", y)); y += 1
red_yellow = lambda bad, warn: [{"color": "red", "value": None}, {"color": "yellow", "value": bad},
{"color": "green", "value": warn}]
panels += [
stat("Доступность (SLO 99%)", "messenger:slo_availability:ratio30m", "percentunit", 0, y,
desc="1 - 5xx/все бизнес-запросы messenger-api за 30 мин. Health-пробы не инструментированы.",
thresholds=red_yellow(0.99, 0.995), decimals=2),
stat("Латентность ≤500 ms (SLO 95%)", "messenger:slo_latency:ratio30m", "percentunit", 4, y,
desc="Доля запросов в bucket le=0.5 s за 30 мин.", thresholds=red_yellow(0.95, 0.97), decimals=2),
stat("Error ratio 5xx (2m)", "messenger:http_error_ratio:rate2m", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}], decimals=2),
stat("Старейшее задание", 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "s", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 60}]),
{"id": next(_ids), "type": "alertlist", "title": "Алерты Prometheus (firing/pending)",
"gridPos": {"x": 16, "y": y, "w": 8, "h": 8},
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [],
"groupMode": "default", "maxItems": 20, "sortOrder": 1, "stateFilter":
{"firing": True, "pending": True, "noData": False, "normal": False, "error": True},
"viewMode": "list", "datasource": "Prometheus"}},
]
y += 4
panels += [
stat("RPS API", "messenger:http_requests:rate2m", "reqps", 0, y, decimals=1),
stat("p95 API (5m)", "messenger:http_latency_p95:rate5m", "s", 4, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
stat("Загрузка worker", "messenger:worker_saturation:ratio", "percentunit", 8, y,
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 0.9}]),
stat("Ожидают пул БД", "max(db_client_connection_pending_requests)", "short", 12, y,
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}]),
]
y += 4
# ------------------------------------------------------------------ API
panels.append(row("API (messenger-api, все реплики)", y)); y += 1
rate_count = "rate(%s_count{%s}[$__rate_interval])" % (HTTP, API)
panels += [
ts("RPS по маршруту и методу", [("sum by (http_request_method, http_route) (%s)" % rate_count,
"{{http_request_method}} {{http_route}}")], "reqps", 0, y,
desc="Counter = _count гистограммы http.server.request.duration; route — шаблон, не реальный UUID."),
ts("RPS по классу статуса", [(
'sum by (status_class) (label_replace(%s, "status_class", "${1}xx", "http_response_status_code", "(\\\\d)..")) ' % rate_count,
"{{status_class}}")], "reqps", 12, y, stack=True),
]
y += 8
bucket = "sum by (le) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)
panels += [
ts("Латентность API p50/p95/p99 (из histogram buckets)", [
(q(0.5, bucket), "p50", "A"), (q(0.95, bucket), "p95", "B"), (q(0.99, bucket), "p99", "C")], "s", 0, y,
desc="histogram_quantile по сумме buckets всех реплик; не среднее и не усреднение p99 реплик."),
ts("Доля ошибок 5xx", [
('(sum(rate(%s_count{%s,http_response_status_code=~"5.."}[$__rate_interval])) or vector(0)) / sum(%s)' % (HTTP, API, rate_count),
"5xx / все", "A")], "percentunit", 12, y,
desc="Числитель: 5xx; знаменатель: все бизнес-запросы (health исключён). 4xx не считаются отказом."),
]
y += 8
panels += [
ts("p95 по маршруту", [(q(0.95, "sum by (le, http_route) (rate(%s_bucket{%s}[$__rate_interval]))" % (HTTP, API)),
"{{http_route}}")], "s", 0, y),
ts("Запросы в обработке", [("sum by (instance) (http_server_active_requests{%s})" % API, "{{instance}}")],
"short", 12, y),
]
y += 8
# ------------------------------------------------------------------ worker and queue
panels.append(row("Очередь и worker", y)); y += 1
jb = "sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome=\"succeeded\"}[$__rate_interval]))"
rb = "sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[$__rate_interval]))"
panels += [
ts("Jobs по исходу (в сек.)", [("sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"{{outcome}}")], "ops", 0, y, stack=True,
desc="succeeded / retry / failed / noop / lost / dlq / requeued; job_type=attachment.process.v1"),
ts("Длительность job (succeeded) p50/p95/p99", [(q(0.5, jb), "p50", "A"), (q(0.95, jb), "p95", "B"),
(q(0.99, jb), "p99", "C")], "s", 12, y),
]
y += 8
panels += [
ts("Глубина очереди (из PostgreSQL)", [("max by (queue, state) (messenger_queue_depth)", "{{queue}} {{state}}")],
"short", 0, y, desc="max() по репликам API: каждая публикует одно и то же значение из БД."),
ts("Возраст старейшей job", [('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})', "oldest age")],
"s", 12, y, desc="От durable acceptance (COMMIT job) до сейчас; это backlog, не processing latency.",
extra={"custom": {"lineWidth": 2, "fillOpacity": 10, "showPoints": "never",
"thresholdsStyle": {"mode": "line"}},
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": None},
{"color": "red", "value": 60}]}}),
]
y += 8
panels += [
ts("Время до ready p50/p95/p99", [(q(0.5, rb), "p50", "A"), (q(0.95, rb), "p95", "B"), (q(0.99, rb), "p99", "C")],
"s", 0, y, desc="messenger.attachment.time_to_ready: от 202 (COMMIT job) до COMMIT ready."),
ts("Насыщение worker: занято / ёмкость", [
("sum(messenger_worker_busy)", "busy", "A"), ("sum(messenger_worker_capacity)", "capacity", "B"),
("messenger:worker_saturation:ratio", "saturation", "C")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ resources
panels.append(row("Ресурсы: пул БД, CPU, RAM", y)); y += 1
panels += [
ts("Пул БД: used / idle / waiting", [
("sum by (service_name, db_client_connection_state) (db_client_connection_count)",
"{{service_name}} {{db_client_connection_state}}", "A"),
("sum by (service_name) (db_client_connection_pending_requests)", "{{service_name}} waiting", "B"),
("sum by (service_name) (db_client_connection_max)", "{{service_name}} max", "C")], "short", 0, y),
ts("CPU процессов (ядра)", [
("sum by (service_name, instance) (rate(process_cpu_time_seconds_total[$__rate_interval]))",
"{{service_name}} {{instance}}", "A"),
("rate(otelcol_process_cpu_seconds_total[$__rate_interval])", "otel-collector", "B")], "percentunit", 12, y,
desc="worker: self + завершённые дочерние процессы обработки изображений."),
]
y += 8
panels += [
ts("RSS процессов", [("process_memory_usage_bytes", "{{service_name}} {{instance}}", "A"),
("otelcol_process_memory_rss_bytes", "otel-collector", "B")], "bytes", 0, y),
ts("Telemetry pipeline: принято / отправлено / ошибки (items/s)", [
("sum(rate(otelcol_receiver_accepted_spans_total[$__rate_interval]))", "accepted spans", "A"),
("sum(rate(otelcol_receiver_accepted_log_records_total[$__rate_interval]))", "accepted log records", "E"),
("sum(rate(otelcol_receiver_accepted_metric_points_total[$__rate_interval]))", "accepted metric points", "F"),
("sum by (exporter) (rate({__name__=~\"otelcol_exporter_sent_(spans|log_records|metric_points)_total\"}[$__rate_interval]))",
"sent {{exporter}}", "B"),
# Separate queries: rate() drops __name__, so a regex over several failed_* counters collides once they all exist.
("sum(rate(otelcol_exporter_send_failed_spans_total[$__rate_interval])) or vector(0)", "send failed spans", "C"),
("sum(rate(otelcol_exporter_enqueue_failed_spans_total[$__rate_interval])) or vector(0)", "dropped spans (queue full)", "G"),
("sum(rate(otelcol_exporter_send_failed_log_records_total[$__rate_interval])) or vector(0)", "send failed logs", "H"),
("sum(rate(otelcol_exporter_enqueue_failed_log_records_total[$__rate_interval])) or vector(0)", "dropped logs (queue full)", "I"),
("sum by (exporter) (otelcol_exporter_queue_size)", "queue {{exporter}}", "D")], "short", 12, y),
]
y += 8
# ------------------------------------------------------------------ logs and traces
panels.append(row("Логи и трассы", y)); y += 1
panels += [
{"id": next(_ids), "type": "logs", "title": "Логи API и worker (warning/error)", "datasource": LOKI,
"gridPos": {"x": 0, "y": y, "w": 12, "h": 10},
"options": {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True},
"targets": [{"datasource": LOKI, "refId": "A",
"expr": '{service_name=~"messenger-api|messenger-worker"} | severity_text=~"WARN.*|ERROR"'}]},
{"id": next(_ids), "type": "table", "title": "Трассы с ошибками или медленнее 500 ms (TraceQL)", "datasource": TEMPO,
"gridPos": {"x": 12, "y": y, "w": 12, "h": 10},
"targets": [{"datasource": TEMPO, "refId": "A", "queryType": "traceql", "limit": 20,
"query": '{resource.service.name=~"messenger-.*" && (status=error || duration>500ms)}'}]},
]
y += 10
dashboard = {
"uid": "messenger-lab6", "title": "Messenger — наблюдаемость (lab6)", "tags": ["messenger", "lab6"],
"timezone": "browser", "schemaVersion": 39, "version": 1, "editable": False, "refresh": "10s",
"time": {"from": "now-30m", "to": "now"}, "graphTooltip": 1, "panels": panels,
"annotations": {"list": [{"builtIn": 1, "datasource": {"type": "grafana", "uid": "-- Grafana --"},
"enable": True, "hide": True, "name": "Annotations & Alerts", "type": "dashboard"}]},
"templating": {"list": []},
}
OUT.write_text(json.dumps(dashboard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("wrote %s (%d panels)" % (OUT.relative_to(OUT.parents[3]), len(panels)))
# Human-readable list of every panel: datasource, query and unit (kept in sync with the JSON by this script).
UNITS = {"reqps": "запросы/с", "percentunit": "доля (0..1)", "s": "секунды", "ops": "операции/с", "short": "штуки",
"bytes": "байты"}
DOC = OUT.parents[3] / "docs/dashboard.md"
lines = ["# Панели dashboard «%s»" % dashboard["title"], "",
"Сгенерировано `scripts/gen_dashboard.py` вместе с `infra/grafana/dashboards/messenger.json`. "
"`$__rate_interval` подставляет Grafana (не меньше 4 × scrape/export interval).", ""]
for panel in panels:
if panel["type"] == "row":
lines += ["## " + panel["title"], ""]
continue
unit = panel.get("fieldConfig", {}).get("defaults", {}).get("unit", "")
lines.append("**%s** (%s%s)" % (panel["title"], panel["type"], ", единица: " + UNITS.get(unit, unit) if unit else ""))
if panel.get("description"):
lines.append("— " + panel["description"])
lines.append("")
if panel["type"] == "alertlist":
lines += ["Правила Prometheus в состоянии pending/firing (`infra/prometheus/rules/messenger.yml`).", ""]
for t in panel.get("targets", []):
query = t.get("expr") or t.get("query")
legend = t.get("legendFormat", "")
lines += ["```%s" % ("promql" if t["datasource"]["type"] == "prometheus" else ""),
query + (" # " + legend if legend and legend != panel["title"] else ""), "```"]
lines.append("")
DOC.write_text("\n".join(lines), encoding="utf-8")
print("wrote docs/dashboard.md")
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# Creates a local .env and random credentials in secrets/ (both are git-ignored). Prints no secret values.
# Safe to repeat: existing files are kept, so running it on a live stand does not change its passwords.
set -eu
cd "$(dirname "$0")/.."
[ -f .env ] || cp .env.example .env
umask 077
mkdir -p secrets
cd secrets
put() { # put <file> <value>
[ -f "$1" ] || printf '%s' "$2" > "$1"
}
random() { python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')"; }
put postgres_password "$(random)"
put minio_root_user messenger_root
put minio_root_password "$(random)"
put s3_access_key_id messenger_app
put s3_secret_access_key "$(random)"
put rabbitmq_password "$(random)"
put grafana_admin_password "$(random)"
put cache_url valkey://valkey:6379/0
put database_url "postgresql://messenger:$(cat postgres_password)@postgres:5432/messenger"
put broker_url "amqp://messenger:$(cat rabbitmq_password)@rabbitmq:5672/messenger"
# Containers run under other uids and read these through Docker secrets (file bind mounts in Compose).
chmod 644 ./*
echo "secrets ready in secrets/ (.env: $( [ -f ../.env ] && echo present ))"
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...).
failed registry of failed jobs (the "DLQ" of business failures)
show <attachment_id> job state + audit trail
replay <attachment_id> --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing
stats counts by state and outbox backlog
Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md).
"""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import jobs # noqa: E402
from app.cursor import fmt # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
def out(value) -> None:
print(json.dumps(value, ensure_ascii=False, indent=2, default=str))
async def main() -> int:
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("failed")
sub.add_parser("stats")
show = sub.add_parser("show")
show.add_argument("attachment_id")
rp = sub.add_parser("replay")
rp.add_argument("attachment_id")
rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)")
args = parser.parse_args()
db = Database()
await db.open()
try:
if args.cmd == "failed":
async with db.connection() as conn:
out(await jobs.failed_jobs(conn))
elif args.cmd == "stats":
async with db.connection() as conn:
cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state")
states = {r["state"]: r["n"] for r in await cur.fetchall()}
cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL")
backlog = (await cur.fetchone())["n"]
out({"jobs": states, "outbox_unpublished": backlog})
elif args.cmd == "show":
async with db.connection() as conn:
cur = await conn.execute(
"SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until,"
" a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id"
" WHERE j.attachment_id = %s", (args.attachment_id,))
job = await cur.fetchone()
cur = await conn.execute(
"SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
(args.attachment_id,))
trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()]
cur = await conn.execute(
"SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts"
" FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id",
(args.attachment_id,))
outbox = await cur.fetchall()
out({"job": job, "audit": trail, "outbox": outbox})
elif args.cmd == "replay":
s3 = S3Storage()
await run_in_threadpool(s3.open)
async def exists(key):
return await run_in_threadpool(s3.exists, key)
result = await db.in_transaction(
lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists))
out(result)
return 0 if result["result"] == "queued" else 3
finally:
await db.close()
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
import hashlib
import os
import pathlib
import sys
import psycopg
ROOT = pathlib.Path(__file__).resolve().parents[1]
MIGRATIONS = ROOT / "migrations"
LOCK_ID = 2026_0004
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE")
if path:
return pathlib.Path(path).read_text(encoding="utf-8").strip()
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
return url
def main() -> None:
files = sorted(MIGRATIONS.glob("*.sql"))
if not files:
sys.exit("no migration files found in %s" % MIGRATIONS)
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY,"
" checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
applied = {row[0]: row[1] for row in rows}
for path in files:
version = path.stem
body = path.read_text(encoding="utf-8")
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
if version in applied:
if applied[version] != checksum:
sys.exit("migration %s changed after it was applied" % version)
print("skip %s" % version)
continue
with conn.transaction():
conn.execute(body)
conn.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
(version, checksum),
)
print("apply %s" % version)
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
print("migrations complete: %d file(s)" % len(files))
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
try:
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
finally:
await db.close()
if __name__ == "__main__":
asyncio.run(main())
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Restart API, LB, worker and every store keeping all volumes: business stores (postgres, valkey, minio, rabbitmq)
# and, since lab6, the telemetry stores (prometheus, loki, tempo) plus collector and grafana.
set -eu
cd "$(dirname "$0")/.."
docker compose restart -t 10 postgres valkey minio rabbitmq
docker compose restart -t 10 prometheus loki tempo otel-collector grafana
docker compose restart -t 25 worker api-1 api-2 nginx
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Every Prometheus query of the provisioned dashboard must parse and return data (after some traffic).
python3 student-tests/check_dashboard.py
"""
import json
import sys
import urllib.error
from pathlib import Path
import obs
ROOT = Path(__file__).resolve().parents[1]
dash = json.loads((ROOT / "infra/grafana/dashboards/messenger.json").read_text(encoding="utf-8"))
bad = 0
for panel in dash["panels"]:
for t in panel.get("targets", []):
if t.get("datasource", {}).get("type") != "prometheus":
continue
expr = t["expr"].replace("$__rate_interval", "2m")
try:
rows = obs.prom(expr)
status = "ok %3d series" % len(rows) if rows else "EMPTY"
except urllib.error.HTTPError as exc:
status, rows = "ERROR %s" % exc.read()[:120], None
if not rows:
bad += 1
print("%-16s %-48s %s" % (status, panel["title"][:48], t.get("legendFormat", "")))
print("panels: %d, prometheus queries without data: %d" % (len(dash["panels"]), bad))
# ---------------------------------------------------------------- the same panels executed BY GRAFANA (/api/ds/query)
# Proves the provisioned datasources work from Grafana itself (Prometheus, Loki and Tempo), with Grafana macros.
import base64 # noqa: E402
import urllib.request # noqa: E402
password = (ROOT / "secrets/grafana_admin_password").read_text().strip()
auth = "Basic " + base64.b64encode(("admin:" + password).encode()).decode()
via_bad = 0
for panel in dash["panels"]:
for t in panel.get("targets", []):
q = dict(t)
q.setdefault("datasource", panel.get("datasource"))
if q["datasource"]["type"] == "tempo":
# Grafana 11 runs TraceQL search in the browser through the datasource proxy, not /api/ds/query
# ("backend TraceQL search queries are not supported"): check exactly that proxy path.
import time as _t
import urllib.parse as _p
url = ("http://127.0.0.1:3000/api/datasources/proxy/uid/tempo/api/search?" +
_p.urlencode({"q": q["query"], "limit": q.get("limit", 20), "start": int(_t.time()) - 1800,
"end": int(_t.time())}))
req = urllib.request.Request(url, headers={"Authorization": auth})
try:
with urllib.request.urlopen(req, timeout=60) as response:
n = len(json.loads(response.read()).get("traces", []))
status = "ok %5d traces" % n if n else "EMPTY"
except urllib.error.HTTPError as exc:
status = "ERROR HTTP %s" % exc.code
if not status.startswith("ok"):
via_bad += 1
print("grafana %-8s %-18s %-48s (datasource proxy)" % ("tempo", status, panel["title"][:48]))
continue
body = json.dumps({"queries": [q], "from": "now-30m", "to": "now"}).encode()
req = urllib.request.Request("http://127.0.0.1:3000/api/ds/query", data=body, method="POST",
headers={"Content-Type": "application/json", "Authorization": auth})
try:
with urllib.request.urlopen(req, timeout=60) as response:
res = json.loads(response.read())["results"][q["refId"]]
frames = res.get("frames", [])
rows = sum(len((f.get("data") or {}).get("values", [[]])[0] or []) for f in frames)
status = "ERROR %s" % res["error"][:100] if res.get("error") else ("ok %5d values" % rows if rows else "EMPTY")
except urllib.error.HTTPError as exc:
status = "ERROR HTTP %s %s" % (exc.code, exc.read()[:100])
if not status.startswith("ok"):
via_bad += 1
print("grafana %-8s %-18s %-48s" % (q["datasource"]["type"], status, panel["title"][:48]))
print("via Grafana: queries without data or with errors: %d" % via_bad)
+139
View File
@@ -0,0 +1,139 @@
"""Query helpers for the telemetry backends (stdlib only). Backends listen on 127.0.0.1 (compose.yaml).
Never prints tokens or passwords; callers print only ids, counts, span names and attributes.
"""
import base64
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
PROM = os.environ.get("PROM_URL", "http://127.0.0.1:9090")
LOKI = os.environ.get("LOKI_URL", "http://127.0.0.1:3100")
TEMPO = os.environ.get("TEMPO_URL", "http://127.0.0.1:3200")
def _get(url, params=None, timeout=15):
if params:
url += "?" + urllib.parse.urlencode(params)
with urllib.request.urlopen(url, timeout=timeout) as response:
return json.loads(response.read())
def prom(expr, at=None):
"""Instant query -> list of (labels, float value)."""
params = {"query": expr}
if at:
params["time"] = at
data = _get(PROM + "/api/v1/query", params)["data"]["result"]
return [(r["metric"], float(r["value"][1])) for r in data]
def prom_scalar(expr, default=0.0, at=None):
rows = prom(expr, at)
return rows[0][1] if rows else default
def prom_range(expr, start, end, step=5):
data = _get(PROM + "/api/v1/query_range", {"query": expr, "start": start, "end": end, "step": step})
return data["data"]["result"]
def alerts():
"""{alertname: state} for pending/firing alerts in Prometheus."""
out = {}
for a in _get(PROM + "/api/v1/alerts")["data"]["alerts"]:
out[a["labels"]["alertname"]] = a["state"]
return out
def loki(query, minutes=15, limit=500):
end = time.time_ns()
data = _get(LOKI + "/loki/api/v1/query_range", {"query": query, "limit": limit, "direction": "forward",
"start": end - minutes * 60 * 10**9, "end": end})
lines = []
for stream in data["data"]["result"]:
for ts, line in stream["values"]:
lines.append({"ts": int(ts), "line": line, **stream["stream"]})
return sorted(lines, key=lambda item: item["ts"])
def tempo_search(traceql, minutes=30, limit=100, start=None, end=None):
end = int(end or time.time() + 60)
start = int(start or end - 60 - minutes * 60)
data = _get(TEMPO + "/api/search", {"q": traceql, "start": start, "end": end, "limit": limit, "spss": 50},
timeout=60)
return data.get("traces", [])
def _hex(value):
return base64.b64decode(value).hex() if value else None
def _attr(value):
for key in ("stringValue", "intValue", "boolValue", "doubleValue"):
if key in value:
return value[key]
return None
def tempo_trace(trace_id):
"""Flat list of spans: service, name, kind, span_id, parent_id, duration_ms, status, attrs (dict)."""
try:
data = _get(TEMPO + "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
return []
raise
spans = []
for batch in data.get("batches", []):
service = {a["key"]: _attr(a["value"]) for a in batch["resource"]["attributes"]}.get("service.name")
for scope in batch.get("scopeSpans", []):
for s in scope["spans"]:
spans.append({
"service": service, "name": s["name"], "kind": s.get("kind", "").replace("SPAN_KIND_", ""),
"span_id": _hex(s["spanId"]), "parent_id": _hex(s.get("parentSpanId")),
"start": int(s["startTimeUnixNano"]),
"duration_ms": (int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"])) / 1e6,
"status": s.get("status", {}).get("code", "").replace("STATUS_CODE_", "") or "UNSET",
"attrs": {a["key"]: _attr(a["value"]) for a in s.get("attributes", [])},
})
return sorted(spans, key=lambda item: item["start"])
def tempo_raw(trace_id):
with urllib.request.urlopen(TEMPO + "/api/traces/" + trace_id, timeout=15) as response:
return response.read().decode("utf-8", "replace")
def print_tree(spans, out=print, skip_db=False):
children = {}
ids = {s["span_id"] for s in spans}
for s in spans:
children.setdefault(s["parent_id"] if s["parent_id"] in ids else None, []).append(s)
def walk(parent, depth):
for s in children.get(parent, []):
if skip_db and s["kind"] == "CLIENT" and s["name"].split()[0] in ("SELECT", "UPDATE", "INSERT", "DELETE"):
continue
extra = {k: v for k, v in s["attrs"].items() if k.startswith(("messenger.", "messaging.message", "db.system",
"http.response.status_code", "rpc.method"))}
out("%s%-8s %-9s %-44s %8.1f ms %-5s %s" % (" " * depth, s["service"].replace("messenger-", ""), s["kind"],
s["name"][:44], s["duration_ms"], s["status"],
json.dumps(extra, ensure_ascii=False)))
walk(s["span_id"], depth + 1)
walk(None, 0)
def wait_for(predicate, seconds=60, interval=2):
deadline = time.monotonic() + seconds
value = None
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(interval)
return value
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time,
so a slow server cannot slow down the generator (no coordinated omission while threads are available).
Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png).
Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500)
are added to GET history with the given percentages, to measure whether such traces survive sampling.
python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \
--output evidence/x.json
Tokens stay in memory; the summary contains counts, percentiles and timestamps only.
"""
import argparse
import http.client
import json
import queue
import random
import sys
import threading
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def percentile(values, p):
if not values:
return None
values = sorted(values)
k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1))
return values[k]
class Load:
def __init__(self, base_url, users=4, threads=64, timeout=30.0):
self.base = urlsplit(base_url)
self.threads = threads
self.timeout = timeout
c = Client(base_url)
self.actors = []
for i in range(users):
a, b = c.user("load%da" % i), c.user("load%db" % i)
chat = c.call("POST", "/api/v1/chats", 201, actor=a,
body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"]
self.actors.append((a["token"], chat["id"]))
self.lock = threading.Lock()
self.results = []
self.local = threading.local()
def conn(self):
c = getattr(self.local, "conn", None)
if c is None:
c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout)
self.local.conn = c
return c
def request(self, method, path, body=None, headers=None):
for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending
conn = self.conn()
try:
conn.request(method, path, body=body, headers=headers or {})
response = conn.getresponse()
response.read()
return response.status, response.getheader("X-Request-Id")
except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc:
conn.close()
self.local.conn = None
if attempt == 2:
return "conn_error:" + type(exc).__name__, None
except (OSError, http.client.HTTPException) as exc:
conn.close()
self.local.conn = None
return "error:" + type(exc).__name__, None
return "error", None
def one(self, kind, fault, scheduled):
token, chat = random.choice(self.actors)
headers = {"Authorization": "Bearer " + token}
if fault:
headers["X-Fault"] = fault
if kind == "post_message":
headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex})
status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat,
json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers)
elif kind == "get_history":
status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers)
else:
headers["Content-Type"] = "image/png"
status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers)
done = time.monotonic()
with self.lock:
self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled,
"at": time.time(), "rid": rid})
def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None):
work = queue.Queue()
start = time.monotonic()
self.started_at = time.time()
def worker():
while True:
item = work.get()
if item is None:
return
self.one(*item)
pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)]
for t in pool:
t.start()
def schedule(r, picker):
if r <= 0:
return
n = 0
while True:
due = start + n / r
if due - start >= duration or (stop_event and stop_event.is_set()):
return
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
work.put(picker() + (due,))
n += 1
def pick_main():
if random.random() < 0.75:
return ("post_message", None)
roll = random.random() * 100
fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None
return ("get_history", fault) # 25% share * 4 = requested share of ALL requests
schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True),
threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)]
for t in schedulers:
t.start()
for t in schedulers:
t.join()
backlog = work.qsize()
for _ in pool:
work.put(None)
for t in pool:
t.join(self.timeout + 5)
self.finished_at = time.time()
return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog)
def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog):
res = self.results
ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400]
plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"]
out = {
"profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate,
"slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads,
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"},
"started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog,
"requests": len(res), "ok": len(ok),
"successful_rps": round(len(ok) / max(duration, 1e-9), 2),
"status_counts": {},
"server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and (
not isinstance(r["status"], int) or r["status"] >= 500)),
"latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None
for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))},
"faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"),
"error": sum(1 for r in res if r["fault"] == "error")},
"by_kind": {},
}
for r in res:
key = str(r["status"])
out["status_counts"][key] = out["status_counts"].get(key, 0) + 1
for kind in ("post_message", "get_history", "upload"):
lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None]
out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind),
"p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None,
"p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None}
return out
def main():
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--base-url", default="http://localhost:8080")
p.add_argument("--rate", type=float, default=40)
p.add_argument("--duration", type=float, default=60)
p.add_argument("--upload-rate", type=float, default=0)
p.add_argument("--slow-pct", type=float, default=0)
p.add_argument("--error-pct", type=float, default=0)
p.add_argument("--threads", type=int, default=64)
p.add_argument("--output")
a = p.parse_args()
load = Load(a.base_url, threads=a.threads)
summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct)
text = json.dumps(summary, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(a.output).write_text(text + "\n", encoding="utf-8")
if __name__ == "__main__":
main()
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB.
Each run:
1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH
2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses)
3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second
4. waits until every accepted upload is ready/failed and computes from attachment_jobs:
queue lag = first_started_at - created_at (time a job waited for a worker)
time to ready = finished_at - attachment.created_at
throughput = finished jobs / (last finish - first arrival)
Rejections (503 queue_full from admission control) are counted, not retried.
python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2
python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30
Results are appended as JSON lines to evidence/track1/results.jsonl.
"""
import argparse
import json
import os
import statistics
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready
OUT = ROOT / "evidence/track1"
SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")}
GEN = r'''
import io, random, sys
from PIL import Image, ImageChops, ImageDraw
w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3]
if fmt == "JPEG": # photo-like noise: realistic JPEG sizes
chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)]
grain = Image.effect_noise((w, h), 12)
img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan])
else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling)
img = Image.linear_gradient("L").resize((w, h)).convert("RGB")
draw, rnd = ImageDraw.Draw(img), random.Random(1)
for _ in range(400):
x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300)
draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256)))
buf = io.BytesIO()
img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6}))
sys.stdout.buffer.write(buf.getvalue())
'''
def image(size):
w, h, fmt = SIZES[size]
cache = OUT / ("fixture-%s.%s" % (size, fmt.lower()))
if not cache.exists():
OUT.mkdir(parents=True, exist_ok=True)
data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt],
cwd=ROOT, capture_output=True, check=True).stdout
cache.write_bytes(data)
return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg"
def set_worker(replicas, concurrency, prefetch, cpus="2.00"):
env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency),
WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus)
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"],
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
def set_admission(max_pending):
env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending))
subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"],
cwd=ROOT, env=env, check=True, capture_output=True, timeout=180)
def parse_mem(text):
value = text.split("/")[0].strip()
for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)):
if value.endswith(unit):
return float(value[: -len(unit)]) * mult
return 0.0
class Sampler(threading.Thread):
def __init__(self, tag):
super().__init__(daemon=True)
self.tag = tag
self.stop = threading.Event()
self.rss, self.pending, self.cpu = [], [], []
def run(self):
while not self.stop.is_set():
ids = compose("ps", "-q", "worker", check_rc=False).split()
if ids:
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids],
capture_output=True, text=True).stdout
rows = [line.split("|") for line in out.splitlines() if "|" in line]
self.rss.append(sum(parse_mem(r[0]) for r in rows))
self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows))
try:
self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')"
% self.tag)[0][0]))
except Exception:
pass
self.stop.wait(1.0)
def pct(values, q):
if not values:
return None
values = sorted(values)
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
def drive(c, actor, chat, body, ctype, rate, duration, tag):
"""Open-loop: request i is sent at t0 + i/rate regardless of earlier responses."""
total = int(rate * duration)
results = {"accepted": 0, "rejected_503": 0, "other_errors": 0}
lock = threading.Lock()
def one(i):
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i)))
try:
r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype})
key = "accepted" if r["status"] == 202 else "rejected_503"
except Exception: # connection errors, timeouts: counted, never retried
key = "other_errors"
with lock:
results[key] += 1
t0 = time.monotonic()
with ThreadPoolExecutor(max_workers=64) as pool:
for i in range(total):
delay = t0 + i / rate - time.monotonic()
if delay > 0:
time.sleep(delay)
pool.submit(one, i)
results["send_seconds"] = round(time.monotonic() - t0, 2)
results["offered"] = total
return results
def wait_drained(tag, seconds):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0])
if left == 0:
return True
time.sleep(1)
return False
def job_metrics(tag):
rows = sql(
"SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at),"
" j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id"
" WHERE a.filename LIKE 'load-%s-%%'" % tag)
done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"]
lag = [s - a for a, s, f in done]
ttr = [f - a for a, s, f in done]
span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0
return {
"succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"),
"retries": sum(int(r[4]) - 1 for r in rows),
"throughput_jobs_s": round(len(done) / span, 3) if span else None,
"lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0),
"ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0),
}
def record(summary):
OUT.mkdir(parents=True, exist_ok=True)
with open(OUT / "results.jsonl", "a") as handle:
handle.write(json.dumps(summary, ensure_ascii=False) + "\n")
log(json.dumps(summary, ensure_ascii=False))
def experiment(args, label, max_pending=None):
c = Client(timeout=60)
wait_http_ready(c)
alice = c.user("load")
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]},
schema="Chat")["json"]
body, ctype = image(args.size)
set_worker(args.replicas, args.concurrency, args.prefetch)
time.sleep(2)
tag = uuid.uuid4().hex[:8]
sampler = Sampler(tag)
sampler.start()
log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % (
label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch))
sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag)
load_end = time.monotonic()
drained = wait_drained(tag, args.drain_timeout)
drain_seconds = round(time.monotonic() - load_end, 2)
sampler.stop.set()
sampler.join()
summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate,
"duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency,
"prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained,
"drain_after_load_s": drain_seconds, **job_metrics(tag),
"pending_max": max(sampler.pending or [0]),
"worker_mem_peak_mib": round(max(sampler.rss or [0]), 1),
"worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None,
"worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1),
"at": time.strftime("%Y-%m-%dT%H:%M:%S")}
return summary, c, alice, chat
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["run", "overload"])
parser.add_argument("--size", choices=sorted(SIZES), default="medium")
parser.add_argument("--rate", type=float, default=4)
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--replicas", type=int, default=1)
parser.add_argument("--concurrency", type=int, default=2)
parser.add_argument("--prefetch", type=int, default=2)
parser.add_argument("--max-pending", type=int, default=30)
parser.add_argument("--drain-timeout", type=float, default=600)
parser.add_argument("--label", default="")
args = parser.parse_args()
if args.mode == "run":
summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch))
record(summary)
return
# Overload: admission limit on, offered rate far above capacity, then recovery.
set_admission(args.max_pending)
try:
summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending)
# Recovery probe: after the backlog drained, a fresh upload must be ready quickly again.
body, ctype = image("small")
t = time.monotonic()
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body,
headers={"Content-Type": ctype}, schema="Attachment")["json"]
while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready":
time.sleep(0.2)
summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2)
record(summary)
finally:
set_admission(200)
if __name__ == "__main__":
main()
+132
View File
@@ -0,0 +1,132 @@
"""Helpers for student resilience tests: the public HTTP client plus docker compose / psql / mc access.
Standard library only (Python 3.9+). Images that need Pillow are generated inside the API container.
Never prints tokens or secrets.
"""
import json
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check, run # noqa: E402,F401
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def log(*parts):
print(time.strftime("%H:%M:%S"), *parts, flush=True)
def compose(*args, check_rc=True, capture=True, timeout=180):
done = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=capture, text=True, timeout=timeout)
if check_rc and done.returncode != 0:
raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), (done.stderr or "")[-300:]))
return done.stdout if capture else ""
def started_at(*services):
"""Container name -> State.StartedAt, proof that a restart really happened."""
ids = compose("ps", "-q", *services).split()
out = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids],
capture_output=True, text=True).stdout
return "; ".join(line.strip("/")[:60] for line in out.splitlines())
def sql(query):
"""Run a read query in postgres; returns rows as lists of strings (tab separated)."""
out = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t", "-c", query)
return [line.split("\t") for line in out.splitlines() if line]
def s3_keys(prefix):
"""Object keys starting with prefix, listed with the root credentials inside a throwaway minio-init container."""
wanted, prefix = prefix, prefix[:prefix.rfind("/") + 1]
script = ('mc alias set l http://minio:9000 "$(cat /run/secrets/minio_root_user)" '
'"$(cat /run/secrets/minio_root_password)" >/dev/null && '
'mc ls --recursive --json l/messenger-attachments/%s || true' % prefix)
out = compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "minio-init", "-c", script)
keys = []
for line in out.splitlines():
try:
item = json.loads(line)
except ValueError:
continue
if item.get("key"):
keys.append(prefix + item["key"])
return sorted(k for k in keys if k.startswith(wanted))
def queue_depth(queue="attachments.process"):
out = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "--vhost", "messenger", "list_queues",
"name", "messages", "messages_ready", "messages_unacknowledged")
for line in out.splitlines():
parts = line.split()
if parts and parts[0] == queue:
return {"messages": int(parts[1]), "ready": int(parts[2]), "unacked": int(parts[3])}
return None
def wait_queue(min_ready=1, seconds=30):
"""Quorum queue counters are refreshed by the stats emitter (~5 s), so poll."""
deadline = time.monotonic() + seconds
depth = None
while time.monotonic() < deadline:
depth = queue_depth()
if depth and depth["ready"] >= min_ready:
return depth
time.sleep(1)
return depth
def jobs_cli(*args):
done = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "scripts/jobs.py", *args],
cwd=ROOT, capture_output=True, text=True, timeout=60)
return done.returncode, json.loads(done.stdout) if done.stdout.strip() else None
def wait_http_ready(c, seconds=120):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
return
except Failure:
time.sleep(1)
raise Failure("API not ready after %ds" % seconds)
def upload(c, actor, chat, body=SAMPLE, filename="sample.png", ctype="image/png", expected=202):
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe=""))
schema = "Attachment" if expected == 202 else None
return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, schema=schema)["json"]
def get(c, actor, att_id):
return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"]
def wait_status(c, actor, att_id, statuses, seconds=120):
deadline = time.monotonic() + seconds
seen = []
while time.monotonic() < deadline:
att = get(c, actor, att_id)
if not seen or seen[-1] != att["status"]:
seen.append(att["status"])
if att["status"] in statuses:
return att, seen
time.sleep(0.5)
raise Failure("attachment %s did not reach %s in %ds (seen %s)" % (att_id, statuses, seconds, seen))
def objects_of(att_id):
return s3_keys("attachments/%s/" % att_id) + s3_keys("staging/%s" % att_id)
def unique_name(fault=None):
tag = uuid.uuid4().hex[:8]
return ("fault-%s-%s.png" % (fault, tag)) if fault else ("plain-%s.png" % tag)
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Bounded buffering at the limit: Tempo stopped under 100 req/s until the exporter queue is FULL.
Expected: queue_size stops at capacity, extra spans are dropped and counted, collector RSS stays under the container
limit, API has no unexpected errors. python3 student-tests/test_exporter_queue_full.py (~5 min)
"""
import threading
import time
import obs
from openloop import Load
from stand import check, compose, log, run
from test_incidents import docker_mem
def value(expr):
return obs.prom_scalar("sum(%s)" % expr)
def main():
load = Load("http://localhost:8080", users=4, threads=96)
stop, summary = threading.Event(), {}
thread = threading.Thread(target=lambda: summary.update(load.run(100, 900, 1, stop_event=stop)), daemon=True)
thread.start()
time.sleep(20)
q = 'otelcol_exporter_queue_size{exporter="otlp/tempo"}'
capacity = value('otelcol_exporter_queue_capacity{exporter="otlp/tempo"}')
dropped0, refused0 = value("otelcol_exporter_enqueue_failed_spans_total"), value("otelcol_receiver_refused_spans_total")
compose("stop", "-t", "5", "tempo")
t0 = time.time()
log("tempo stopped; 100 req/s + 1 upload/s; exporter queue capacity=%.0f batches" % capacity)
peak_q, mems = 0, []
try:
for _ in range(16):
time.sleep(12)
size, mem = value(q), docker_mem("otel-collector")[0]
peak_q, _ = max(peak_q, size), mems.append(mem)
log(" +%3d s queue=%.0f/%.0f collector_rss=%s dropped=%.0f refused=%.0f" % (
time.time() - t0, size, capacity, mem, value("otelcol_exporter_enqueue_failed_spans_total") - dropped0,
value("otelcol_receiver_refused_spans_total") - refused0))
dropped = value("otelcol_exporter_enqueue_failed_spans_total") - dropped0
refused = value("otelcol_receiver_refused_spans_total") - refused0
finally:
compose("start", "tempo")
stop.set()
thread.join(60)
log("result: peak queue=%.0f/%.0f, dropped by full queue=%.0f spans, refused by memory_limiter=%.0f spans, peak RSS=%s MiB"
% (peak_q, capacity, dropped, refused, max(float(m.replace("MiB", "")) for m in mems)))
log("API: requests=%s ok=%s unexpected_errors=%s p99=%s ms" % (
summary["requests"], summary["ok"], summary["server_errors_unexpected"], summary["latency_ms_plain_ok"]["p99"]))
check(peak_q <= capacity, "queue must stay within capacity")
check(peak_q >= capacity * 0.95 or refused > 0, "the limit must really be reached")
check(dropped + refused > 0, "overflow must be counted")
check(summary["server_errors_unexpected"] == 0, "API must be unaffected")
print("PASS: exporter buffering is bounded, overflow is dropped and counted, API unaffected")
if __name__ == "__main__":
run(main)
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Grade 4 fault-injection suite (needs FAULT_INJECTION=1 on the stand, see REPORT.md).
Each scenario uploads a file whose display name selects an injected fault (app/faults.py) and checks the
recovery through the public API plus DB/S3/broker state:
A api_crash_after_commit crash between COMMIT(job+outbox) and publish -> another replica's relay publishes
B worker_crash_after_put crash after S3 writes, before result COMMIT -> lease expiry -> same keys, one result
C worker_crash_after_commit crash after COMMIT, before ack -> redelivery is a no-op
D transient3 3 transient errors -> retries with growing jittered backoff -> ready
E permanent permanent error -> failed at attempt 1, no retry, content/attach 409
F transient6 attempts exhausted -> failed/retry_exhausted -> registry -> explicit replay -> ready;
replay of a ready job is refused
G worker_hang hung worker (no heartbeats) -> other worker takes over, hung one is fenced
H poison message invalid broker message -> DLQ, not retried forever
I broker down RabbitMQ stopped: upload still 202 (job waits in outbox) -> broker back -> ready
usage: python3 student-tests/test_failure_windows.py [A B C ...]
"""
import sys
import time
from stand import (Client, Failure, check, compose, get, jobs_cli, log, objects_of, queue_depth, run, sql, upload,
unique_name, wait_http_ready, wait_status)
def audit(att_id):
return sql("SELECT action, actor, attempt, coalesce(detail->>'error_code',''), coalesce(detail->>'delay_seconds','')"
" FROM job_audit WHERE attachment_id='%s' ORDER BY id" % att_id)
def show_audit(att_id):
for action, actor, attempt, code, delay in audit(att_id):
log(" audit attempt=%s %-16s %-28s %s %s" % (attempt, action, actor[:28], code, ("delay=%ss" % delay) if delay else ""))
def one_result(att_id):
variants = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att_id)[0][0]
succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att_id)[0][0]
keys = objects_of(att_id)
check(variants == "3", "exactly one set of 3 variant rows, got %s" % variants)
check(succeeded == "1", "exactly one committed success, got %s" % succeeded)
check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys),
"original + 3 variants, staging cleaned: %s" % keys)
def ready_and_single(c, actor, att_id, seconds=120):
att, seen = wait_status(c, actor, att_id, ("ready", "failed"), seconds)
check(att["status"] == "ready", "expected ready, got %s/%s" % (att["status"], att["error_code"]))
log(" statuses:", " -> ".join(seen))
show_audit(att_id)
one_result(att_id)
return att
def scenario_a(c, alice, bob, chat):
name = unique_name("api_crash_after_commit")
try:
upload(c, alice, chat, filename=name, expected=[202, 500, 502, 503])
except Failure as exc:
log(" client saw:", exc)
wait_http_ready(c)
rows = sql("SELECT id FROM attachments WHERE filename='%s'" % name)
check(len(rows) == 1, "committed attachment must exist after API crash")
att_id = rows[0][0]
log(" committed attachment %s survived the API crash" % att_id)
ready_and_single(c, bob, att_id)
pub = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
" WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att_id)[0][0]
check(pub == "1", "outbox row published exactly once by the relay")
def scenario_b(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("worker_crash_after_put"))
ready_and_single(c, bob, att["id"])
actions = [r[0] for r in audit(att["id"])]
check("retry_scheduled" in actions, "lease expiry must schedule a new attempt")
def scenario_c(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("worker_crash_after_commit"))
ready_and_single(c, bob, att["id"])
time.sleep(8)
logs = compose("logs", "--since", "60s", "worker")
check("decision=done" in logs, "redelivered message after crash must be a no-op")
one_result(att["id"])
def scenario_d(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("transient3"))
ready_and_single(c, bob, att["id"])
delays = [float(r[4]) for r in audit(att["id"]) if r[0] == "retry_scheduled"]
check(len(delays) == 3, "three retries expected, got %s" % delays)
bounds = [(1, 2), (2, 4), (4, 8)]
check(all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "equal-jitter backoff bounds %s" % delays)
log(" backoff delays:", delays)
def scenario_e(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("permanent"))
failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 60)
check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", "permanent -> failed/invalid_image")
check(failed["variants"] == [], "failed attachment has no variants")
show_audit(att["id"])
check([r[0] for r in audit(att["id"])].count("claimed") == 1, "permanent errors are not retried")
c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "k-" + att["id"]})
def scenario_f(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("transient6"))
failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 120)
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "exhausted -> retry_exhausted")
attempts = sql("SELECT attempt, max_attempts FROM attachment_jobs WHERE attachment_id='%s'" % att["id"])[0]
check(attempts == ["5", "5"], "bounded: stopped at attempt 5 of 5, got %s" % attempts)
rc, registry = jobs_cli("failed")
check(any(r["attachment_id"] == att["id"] for r in registry), "failed job listed in operator registry")
log(" failed registry contains the job; replay")
rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite")
check(rc == 0 and result["result"] == "queued", "replay must queue: %s" % result)
ready_and_single(c, bob, att["id"], 120)
rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite")
check(rc == 3 and result["result"] == "refused", "replay of a ready job must be refused: %s" % result)
log(" second replay refused:", result["reason"])
one_result(att["id"])
def scenario_g(c, alice, bob, chat):
att = upload(c, alice, chat, filename=unique_name("worker_hang"))
ready_and_single(c, bob, att["id"], 150)
rows = audit(att["id"])
check(any(r[0] == "retry_scheduled" and r[3] == "lease_expired" for r in rows), "lease expiry detected")
time.sleep(30) # the hung worker wakes up and must be fenced
logs = compose("logs", "--since", "180s", "worker")
check("lease_lost" in logs or "job_result_not_committed" in logs, "hung worker must be fenced by the lease")
one_result(att["id"])
POISON = ("import asyncio, aio_pika\n"
"from app import config, broker\n"
"async def m():\n"
" conn = await broker.connect(); ch = await conn.channel(publisher_confirms=True)\n"
" ex, _ = await broker.declare(ch)\n"
" await ex.publish(aio_pika.Message(b'{not json', delivery_mode=2), routing_key=broker.ROUTING_KEY)\n"
" await conn.close()\n"
"asyncio.run(m())\n")
def scenario_h(c, alice, bob, chat):
before = (queue_depth("attachments.dlq") or {}).get("messages", 0)
compose("exec", "-T", "api-1", "python", "-c", POISON)
deadline = time.monotonic() + 30
after = before
while time.monotonic() < deadline and after <= before:
time.sleep(1)
after = (queue_depth("attachments.dlq") or {}).get("messages", 0)
check(after == before + 1, "poison message must land in attachments.dlq (%s -> %s)" % (before, after))
log(" DLQ depth %s -> %s" % (before, after))
def scenario_i(c, alice, bob, chat):
compose("stop", "-t", "10", "rabbitmq")
try:
att = upload(c, alice, chat, filename=unique_name())
check(att["status"] == "queued", "upload accepted while broker is down")
c.message(alice, chat, "", attachments=[att["id"]])
time.sleep(3)
pending = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
" WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0]
check(pending == "1", "job must wait in the outbox, got %s" % pending)
check(get(c, bob, att["id"])["status"] == "queued", "still queued while broker is down")
log(" broker down: upload 202, outbox holds 1 unpublished job")
finally:
compose("start", "rabbitmq")
ready_and_single(c, bob, att["id"], 120)
SCENARIOS = {"A": scenario_a, "B": scenario_b, "C": scenario_c, "D": scenario_d, "E": scenario_e,
"F": scenario_f, "G": scenario_g, "H": scenario_h, "I": scenario_i}
def main():
c = Client(timeout=30)
alice, bob, eve, chat = c.bootstrap()
picked = sys.argv[1:] or list(SCENARIOS)
for key in picked:
log("scenario %s: %s" % (key, SCENARIOS[key].__name__))
started = time.monotonic()
SCENARIOS[key](c, alice, bob, chat)
log("scenario %s PASS (%.1fs)" % (key, time.monotonic() - started))
print("PASS: failure windows %s" % " ".join(picked))
if __name__ == "__main__":
run(main)
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""Level 4 controlled incidents, diagnosed ONLY from stored telemetry (Prometheus / Loki / Tempo), with alert
firing -> resolved timings. A background open-loop traffic (messages + history, 2% slow handler requests) runs all
the time so that error ratios have a denominator.
A. worker stopped -> MessengerQueueLagGrowing firing; start worker -> backlog drains -> resolved
B. DB table lock (slow DB) -> MessengerApiHighErrorRate firing; trace shows the INSERT span waiting 5 s on the lock
(vs. an X-Fault: slow request: long handler, short DB spans); unlock -> resolved
C. Tempo + Loki stopped -> API keeps working, collector queue bounded, drops counted, TelemetryExportFailing;
collector stopped -> API keeps working, SDK queues bounded (RSS flat)
python3 student-tests/test_incidents.py [--only A,B,C] (stack running, FAULT_INJECTION=1; ~15 min)
"""
import argparse
import json
import subprocess
import threading
import time
from pathlib import Path
import obs
from openloop import Load
from stand import Client, check, compose, log, run, upload, wait_status
ROOT = Path(__file__).resolve().parents[1]
RESULTS = {}
def docker_mem(service):
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}"],
capture_output=True, text=True).stdout
for line in out.splitlines():
name, mem, cpu = line.split("\t")
if name.startswith("mtusi-lab6-%s-" % service):
return mem.split(" / ")[0], cpu
return "stopped", "-"
def wait_alert(name, want, seconds, t0):
"""want='firing' waits for firing; want='resolved' waits until the alert is gone. Returns seconds since t0."""
deadline = time.monotonic() + seconds
pending_at = None
while time.monotonic() < deadline:
state = obs.alerts().get(name)
if state == "pending" and pending_at is None:
pending_at = time.time() - t0
log(" %s pending (+%.0f s)" % (name, pending_at))
if (want == "firing" and state == "firing") or (want == "resolved" and state is None):
elapsed = time.time() - t0
log(" %s %s (+%.0f s)" % (name, want, elapsed))
return elapsed, pending_at
time.sleep(5)
raise AssertionError("%s did not become %s within %d s" % (name, want, seconds))
def psql(query):
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query)
# ------------------------------------------------------------------------------------------------ A
def incident_worker(c, alice, chat):
log("=== A. worker stopped (queue lag)")
t0 = time.time()
compose("stop", "-t", "20", "worker")
log("worker stopped at %s" % time.strftime("%H:%M:%S"))
atts = [upload(c, alice, chat, filename="incident-a.png") for _ in range(3)]
log("3 uploads accepted (202) while worker is down: %s" % [a["id"][:8] for a in atts])
firing, pending = wait_alert("MessengerQueueLagGrowing", "firing", 300, t0)
log("diagnosis from telemetry:")
age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})')
depth = {m["state"]: v for m, v in obs.prom('max by (state) (messenger_queue_depth{queue="attachments.process"})')}
processed = obs.prom_scalar('sum(rate(messenger_jobs_processed_total[1m]))')
log(" metrics: oldest job age=%.0f s, depth=%s, worker jobs/s over 1m=%.3f" % (age, depth, processed))
since = int((time.time() - t0) / 60) + 1
worker_lines = obs.loki('{service_name="messenger-worker"} | event=~"job_.*"', minutes=since)
worker_lines = [l for l in worker_lines if l["ts"] / 1e9 > t0 + 25]
stop_lines = obs.loki('{service_name="messenger-worker"} | event=~"worker_stop.*"', minutes=since + 1)
log(" logs: worker job_* events after stop: %d; last worker events: %s" % (
len(worker_lines), [l.get("event") for l in stop_lines[-2:]]))
traces = obs.tempo_search('{span.messenger.attachment_id="%s"}' % atts[0]["id"], minutes=since + 1)
spans = obs.tempo_trace(traces[0]["traceID"]) if traces else []
kinds = sorted({s["kind"] for s in spans if s["kind"] != "CLIENT"})
log(" trace of upload %s: span kinds %s -> published, no CONSUMER: the job waits in the queue" % (
atts[0]["id"][:8], kinds))
check("PRODUCER" in kinds and "CONSUMER" not in kinds, "trace must stop at the producer while worker is down")
t1 = time.time()
compose("start", "worker")
log("worker started (+%.0f s)" % (t1 - t0))
for att in atts:
wait_status(c, alice, att["id"], ("ready",), 120)
drained = time.time() - t1
log("backlog drained: all 3 ready %.1f s after worker start" % drained)
resolved, _ = wait_alert("MessengerQueueLagGrowing", "resolved", 300, t1)
RESULTS["A_worker_stopped"] = {"alert_pending_s": pending, "alert_firing_s": round(firing), "age_at_firing_s":
round(age), "drain_after_restart_s": round(drained, 1),
"resolved_after_restart_s": round(resolved), "lost_jobs": 0}
# ------------------------------------------------------------------------------------------------ B
def incident_db(c):
log("=== B. DB lock on messages (slow/unavailable DB for writes and history reads)")
t0 = time.time()
locker = subprocess.Popen(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
log("ACCESS EXCLUSIVE lock taken on table messages at %s" % time.strftime("%H:%M:%S"))
try:
firing, pending = wait_alert("MessengerApiHighErrorRate", "firing", 300, t0)
log("diagnosis from telemetry:")
ratio = obs.prom_scalar("messenger:http_error_ratio:rate2m")
p95 = obs.prom_scalar("messenger:http_latency_p95:rate5m")
by_route = obs.prom('sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count'
'{service_name="messenger-api",http_response_status_code=~"5.."}[1m]))')
waiting = obs.prom_scalar("max(db_client_connection_pending_requests)")
used = obs.prom_scalar('max(db_client_connection_count{service_name="messenger-api",db_client_connection_state="used"})')
log(" metrics: 5xx ratio=%.1f%%, p95=%.2f s, pool used(max replica)=%.0f waiting=%.0f" % (
ratio * 100, p95, used, waiting))
for m, v in by_route:
log(" 5xx/s %-40s %s %.2f" % (m.get("http_route"), m.get("http_response_status_code"), v))
db_err = obs.tempo_search('{resource.service.name="messenger-api" && span.db.operation.name="INSERT"'
' && status=error}', minutes=5, limit=5)
check(db_err, "no trace with a failed DB span")
spans = obs.tempo_trace(db_err[0]["traceID"])
log(" trace %s (failed request):" % db_err[0]["traceID"])
obs.print_tree(spans, out=lambda line: print(" " + line))
raw = obs.tempo_raw(db_err[0]["traceID"])
cause = next((name for name in ("LockNotAvailable", "QueryCanceled", "PoolTimeout") if name in raw), "?")
db_span = max((s for s in spans if s["attrs"].get("db.system") == "postgresql"), key=lambda s: s["duration_ms"])
server = [s for s in spans if s["kind"] == "SERVER"][0]
log(" => DB span %s took %.0f ms of %.0f ms request, status %s, exception %s: the time is spent in PostgreSQL"
% (db_span["name"], db_span["duration_ms"], server["duration_ms"], db_span["status"], cause))
slow = obs.tempo_search('{span.messenger.fault="slow" && span.http.route="/api/v1/chats/{chat_id}/messages"}',
minutes=10, limit=50)
contrast = None
for item in slow:
sp = obs.tempo_trace(item["traceID"])
srv = [s for s in sp if s["kind"] == "SERVER"]
dbs = [s["duration_ms"] for s in sp if s["attrs"].get("db.system") == "postgresql"]
if srv and dbs and srv[0]["attrs"].get("http.response.status_code") in ("200", 200) and max(dbs) < 100:
contrast = (item["traceID"], srv[0]["duration_ms"], max(dbs), sum(dbs))
break
if contrast:
log(" contrast: slow HANDLER trace %s: request %.0f ms, longest DB span %.1f ms, all DB %.1f ms"
% contrast)
errors = obs.loki('{service_name="messenger-api"} | severity_text=~"WARN.*|ERROR"', minutes=5)
log(" logs: %d API warning/error records in 5 min, events=%s" % (
len(errors), sorted({l.get("event") for l in errors})))
blocker = psql("SELECT pid, state, wait_event_type, left(query, 60) FROM pg_stat_activity "
"WHERE query LIKE '%LOCK TABLE messages%' AND pid <> pg_backend_pid()").strip()
log(" operator step (pg_stat_activity) finds the blocker: %s" % blocker)
finally:
t1 = time.time()
psql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%LOCK TABLE messages%'"
" AND pid <> pg_backend_pid()")
locker.wait(30)
log("lock released (+%.0f s)" % (t1 - t0))
resolved, _ = wait_alert("MessengerApiHighErrorRate", "resolved", 400, t1)
RESULTS["B_db_lock"] = {"alert_pending_s": pending, "alert_firing_s": round(firing),
"error_ratio_at_firing": round(ratio, 3), "p95_at_firing_s": round(p95, 2),
"db_span_ms": round(db_span["duration_ms"]), "request_ms": round(server["duration_ms"]),
"cause": cause, "resolved_after_unlock_s": round(resolved)}
# ------------------------------------------------------------------------------------------------ C
def counter(expr):
return obs.prom_scalar("sum(%s)" % expr)
def incident_telemetry(load):
log("=== C1. Tempo and Loki stopped (telemetry backends down)")
before_res = len(load.results)
dropped0 = counter('otelcol_exporter_enqueue_failed_spans_total') + counter(
'otelcol_exporter_enqueue_failed_log_records_total')
failed0 = counter('otelcol_exporter_send_failed_spans_total') + counter(
'otelcol_exporter_send_failed_log_records_total')
t0 = time.time()
compose("stop", "-t", "5", "tempo", "loki")
samples = []
for i in range(14): # ~140 s
time.sleep(10)
mem, cpu = docker_mem("otel-collector")
q_tempo = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlp/tempo"})')
q_loki = obs.prom_scalar('sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})')
samples.append({"t": round(time.time() - t0), "collector_mem": mem, "cpu": cpu, "queue_tempo": q_tempo,
"queue_loki": q_loki})
log(" +%3d s collector mem=%s cpu=%s queue tempo=%.0f loki=%.0f (capacity 100 batches)" % (
samples[-1]["t"], mem, cpu, q_tempo, q_loki))
alert_state = obs.alerts().get("TelemetryExportFailing")
log(" alert TelemetryExportFailing during outage: %s" % alert_state)
check(alert_state == "firing", "TelemetryExportFailing must fire while backends are down")
window = [r for r in load.results[before_res:]]
bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)]
log(" API during the outage: %d requests, unexpected errors=%d; alert TelemetryExportFailing=%s" % (
len(window), len(bad), alert_state))
check(not bad, "API must keep working while telemetry backends are down")
compose("start", "tempo", "loki")
obs.wait_for(lambda: subprocess.run(["curl", "-sf", obs.TEMPO + "/ready"], capture_output=True).returncode == 0
and subprocess.run(["curl", "-sf", obs.LOKI + "/ready"], capture_output=True).returncode == 0, 120, 3)
t1 = time.time()
log(" tempo/loki started and ready (+%.0f s)" % (t1 - t0))
fresh = obs.wait_for(lambda: obs.tempo_search('{resource.service.name="messenger-api"}', start=t1,
end=time.time() + 5, limit=20), 90, 5)
log(" new traces after restart visible: %d (export resumed)" % len(fresh or []))
check(fresh, "traces must flow again after the backends are back")
resolved, _ = wait_alert("TelemetryExportFailing", "resolved", 300, t1)
time.sleep(20)
dropped = counter('otelcol_exporter_enqueue_failed_spans_total') + counter(
'otelcol_exporter_enqueue_failed_log_records_total') - dropped0
failed = counter('otelcol_exporter_send_failed_spans_total') + counter(
'otelcol_exporter_send_failed_log_records_total') - failed0
log(" telemetry lost and counted: enqueue_failed(queue full)=%.0f items, send_failed(retries exhausted)=%.0f items"
% (dropped, failed))
RESULTS["C1_backends_down"] = {"outage_s": round(t1 - t0), "api_requests": len(window), "api_unexpected_errors":
len(bad), "collector_samples": samples, "alert": alert_state,
"dropped_items": dropped, "send_failed_items": failed,
"traces_after_restart": len(fresh or []),
"alert_resolved_after_restart_s": round(resolved)}
log("=== C2. Collector stopped (SDK side buffering)")
before_res = len(load.results)
api_before = docker_mem("api-1")[0]
compose("stop", "-t", "5", "otel-collector")
mems = []
for _ in range(6):
time.sleep(10)
mems.append(docker_mem("api-1")[0])
window = load.results[before_res:]
bad = [r for r in window if r["fault"] != "error" and (not isinstance(r["status"], int) or r["status"] >= 500)]
compose("start", "otel-collector")
log(" api-1 RSS before=%s during=%s; %d requests, unexpected errors=%d" % (api_before, mems, len(window), len(bad)))
check(not bad, "API must keep working while the collector is down")
RESULTS["C2_collector_down"] = {"outage_s": 60, "api1_mem_before": api_before, "api1_mem_during": mems,
"api_requests": len(window), "api_unexpected_errors": len(bad)}
def main():
p = argparse.ArgumentParser()
p.add_argument("--only", default="A,B,C")
p.add_argument("--output", default=str(ROOT / "evidence/incidents.json"))
args = p.parse_args()
c = Client()
alice, _bob, _eve, chat = c.bootstrap()
load = Load("http://localhost:8080", users=2, threads=48)
stop = threading.Event()
traffic = threading.Thread(target=lambda: RESULTS.setdefault("traffic", load.run(
10, 3600, 0, slow_pct=2, error_pct=0, stop_event=stop)), daemon=True)
traffic.start()
log("background traffic: 10 req/s open-loop (75% POST message, 25% history, 2% X-Fault: slow)")
time.sleep(30)
try:
if "A" in args.only:
incident_worker(c, alice, chat)
if "B" in args.only:
incident_db(c)
if "C" in args.only:
incident_telemetry(load)
finally:
stop.set()
traffic.join(60)
Path(args.output).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2, default=str) + "\n")
log("summary written to %s" % args.output)
print("PASS: incidents %s diagnosed from telemetry, alerts fired and resolved" % args.only)
if __name__ == "__main__":
run(main)
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks.
1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace:
API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans
2. the same trace_id in Loki for messenger-api and messenger-worker log records
3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id
4. no password / token / message text / filename in logs or traces; no ids in metric labels
5. all metric names used by the dashboard and alert rules exist and have samples
python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1)
"""
import re
import time
import uuid
import obs
from stand import Client, check, log, run, unique_name, upload, wait_status
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query",
"filename", "text", "http_target", "url_path"}
REQUIRED_METRICS = [
"http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total",
"messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket",
"messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy",
"messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests",
"process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total",
"otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size",
]
def main():
c = Client()
alice, bob, _eve, chat = c.bootstrap()
marker = "secret-text-" + uuid.uuid4().hex
sent = c.message(alice, chat, marker)
msg_rid = sent["headers"]["X-Request-Id"]
log("message sent request_id=%s (text marker kept private)" % msg_rid)
# ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace
filename = unique_name("transient1")
att = upload(c, alice, chat, filename=filename)
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
check(ready["status"] == "ready", "upload must become ready after one retry")
log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen)))
def find():
found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"])
if not found:
return None
spans = obs.tempo_trace(found[0]["traceID"])
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
producers = [s for s in spans if s["kind"] == "PRODUCER"]
# Spans of one trace arrive from 3 processes with different batch delays: wait until all are in.
return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any(
s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None
result = obs.wait_for(find, 60)
check(result, "trace of the upload with both attempts not found in Tempo")
found, spans = result
trace_ids = {t["traceID"] for t in found}
check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids)
trace_id = found[0]["traceID"]
log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans)))
obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True)
by_id = {s["span_id"]: s for s in spans}
server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"]
producers = [s for s in spans if s["kind"] == "PRODUCER"]
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span")
check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)")
for cons in consumers:
parent = by_id.get(cons["parent_id"])
check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span")
check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request")
retry_parent = by_id[producers[1]["parent_id"]]
check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry",
"retry producer must be a child of the failed consumer")
check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded",
"attempt 1 ERROR/retry, last attempt succeeded")
worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"}
for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"):
check(op in worker_ops, "worker span %s missing" % op)
check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans),
"worker DB spans missing")
log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links")
# ---------------------------------------------------------------- 2. logs of both services with the trace_id
lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"'
% trace_id) if True] or None, 30)
check(lines, "no Loki records with the trace_id")
services = {l["service_name"] for l in lines}
events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines]
log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events))
check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id")
check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events},
"expected events attachment_queued, job_attempt_failed, job_finished")
sample = lines[0]
log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",)))
# ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace
err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"})
rid = err["headers"]["X-Request-Id"]
check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean")
err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30)
check(err_lines, "500 request_id not found in Loki")
err_trace = err_lines[0].get("trace_id")
log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace))
err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30)
check(err_spans, "trace of the 500 not in Tempo")
srv = [s for s in err_spans if s["kind"] == "SERVER"][0]
check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR"
and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR")
log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % (
srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace)))
msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30)
check(msg_trace, "message request trace not found by request_id")
msg_spans = obs.tempo_trace(msg_trace[0]["traceID"])
log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans]))
# ---------------------------------------------------------------- 4. privacy
secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker,
"filename": filename}
raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace)
logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000))
for what, value in secrets.items():
check(value not in raw, "%s leaked into traces" % what)
check(value not in logs_raw, "%s leaked into logs" % what)
# Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo).
for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG",
"\\u0089PNG"):
check(word not in raw and word not in logs_raw, "%r found in telemetry" % word)
log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from "
"traces and logs")
series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}')
labels = set()
for metric, _ in series:
labels |= set(metric)
for name, value in metric.items():
check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value))
check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS))
log("PASS: %d series, label names: %s" % (len(series), sorted(labels)))
routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")})
log("http_route values (templates only): %s" % routes)
# ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts
time.sleep(12) # one metric export interval
for name in REQUIRED_METRICS:
check(obs.prom(name), "metric %s has no samples" % name)
log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS))
print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)")
if __name__ == "__main__":
run(main)
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env python3
"""Grade 3 demo: worker stopped -> upload 202 -> message with attachment -> restart API + broker (volumes kept)
-> worker started -> ready + thumbnail. Then an explicit duplicate delivery of the finished job is a no-op.
python3 student-tests/test_worker_down.py (stack from REPORT.md must be running)
"""
import hashlib
import time
from stand import (Client, check, compose, started_at, get, log, objects_of, wait_queue, run, sql, upload, wait_http_ready,
wait_status, unique_name, SAMPLE)
def main():
c = Client()
alice, bob, eve, chat = c.bootstrap()
log("stop all workers")
compose("stop", "-t", "20", "worker")
try:
att = upload(c, alice, chat, filename=unique_name())
check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload must be 202 queued")
log("upload 202 queued id=%s" % att["id"])
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment")
log("message %s created with queued attachment" % msg["id"])
c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob)
c.message(bob, chat, "другие сообщения работают")
depth = wait_queue()
log("broker queue while worker is down:", depth)
check(depth and depth["ready"] >= 1, "job must be waiting in the broker")
unpublished = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id"
" WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0]
check(unpublished == "0", "outbox row must be published with confirm")
log("restart api-1 api-2 rabbitmq (volumes kept)")
before = started_at("api-1", "api-2", "rabbitmq")
compose("restart", "-t", "10", "rabbitmq", "api-1", "api-2")
wait_http_ready(c)
after = started_at("api-1", "api-2", "rabbitmq")
log("started before:", before)
log("started after: ", after)
check(before != after, "containers must really restart")
check(get(c, bob, att["id"])["status"] == "queued", "still queued after restart, nothing lost")
depth = wait_queue()
log("broker queue after broker restart:", depth)
check(depth and depth["ready"] >= 1, "durable message must survive broker restart")
finally:
log("start workers")
compose("start", "worker")
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
check(ready["status"] == "ready", "expected ready, got %s" % ready["status"])
log("status sequence observed:", " -> ".join(seen))
names = sorted(v["name"] for v in ready["variants"])
check(names == ["crop", "optimized", "thumbnail"], "expected thumbnail+crop+optimized, got %s" % names)
thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0]
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail 256x160")
body = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"]
check(hashlib.sha256(body).hexdigest() == thumb["sha256"], "thumbnail checksum")
orig = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"]
check(orig == SAMPLE, "original unchanged byte for byte")
keys = objects_of(att["id"])
log("objects:", keys)
check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), "original + 3 variants, staging removed")
log("redelivery: republish the finished job through the outbox (duplicate message)")
sql("INSERT INTO outbox (job_id, payload, available_at, created_at) SELECT o.job_id, o.payload, now(), now()"
" FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='%s' ORDER BY o.id LIMIT 1"
% att["id"])
time.sleep(5)
again = get(c, bob, att["id"])
check(again["status"] == "ready" and again["variants"] == ready["variants"], "duplicate must not change the result")
rows = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att["id"])[0][0]
succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att["id"])[0][0]
check(rows == "3" and succeeded == "1", "one set of variants and one success (rows=%s succeeded=%s)" % (rows, succeeded))
check(len(objects_of(att["id"])) == 4, "no new objects after duplicate")
noop = compose("logs", "--since", "30s", "worker")
check("decision=done" in noop, "worker must log the duplicate as no-op")
log("duplicate acked as no-op: variants=%s successes=%s" % (rows, succeeded))
print("PASS: worker down / API+broker restart / redelivery no-op")
if __name__ == "__main__":
run(main)
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""MessengerWorkerSaturated: capacity is lowered (2 workers x 1 slot, 0.25 CPU each), uploads exceed it for longer than
`for: 5m`; the alert must fire, and resolve after the normal configuration is restored and the backlog drains.
python3 student-tests/test_worker_saturation.py (stack running; ~10 min). Restores WORKER_* defaults at the end.
"""
import json
import os
import subprocess
import threading
import time
from pathlib import Path
import obs
from openloop import Load
from stand import check, log, run
from test_incidents import wait_alert
ROOT = Path(__file__).resolve().parents[1]
ALERT = "MessengerWorkerSaturated"
def worker(env):
done = subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--no-deps", "--force-recreate",
"worker"], cwd=ROOT, env={**os.environ, **env}, capture_output=True, text=True, timeout=240)
check(done.returncode == 0, "worker recreate failed: %s" % done.stderr[-300:])
def main():
result = {}
log("lower capacity: WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 WORKER_CPUS=0.25, 2 replicas")
worker({"WORKER_CONCURRENCY": "1", "WORKER_PREFETCH": "1", "WORKER_CPUS": "0.25"})
load = Load("http://localhost:8080", users=2, threads=32)
stop = threading.Event()
summary = {}
thread = threading.Thread(target=lambda: summary.update(load.run(0, 900, upload_rate=4, stop_event=stop)), daemon=True)
t0 = time.time()
thread.start()
log("uploads: 4/s open-loop (sample.png)")
try:
time.sleep(60)
capacity = obs.prom_scalar("sum(messenger_worker_capacity and (time() - timestamp(messenger_worker_capacity) < 30))")
log("fresh capacity=%.0f slots (stale series of recreated containers are excluded)" % capacity)
check(capacity == 2, "expected 2 slots, got %s" % capacity)
firing, pending = wait_alert(ALERT, "firing", 480, t0)
ratio = obs.prom_scalar("messenger:worker_saturation:ratio")
depth = obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})')
age = obs.prom_scalar('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})')
rate = obs.prom_scalar('sum(rate(messenger_jobs_processed_total{outcome="succeeded"}[2m]))')
p95 = obs.prom_scalar('histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="succeeded"}[5m])))')
others = obs.alerts()
log("at firing: saturation=%.2f, queued=%.0f, oldest age=%.0f s, succeeded=%.2f jobs/s, job p95=%.2f s, alerts=%s"
% (ratio, depth, age, rate, p95, others))
check(ratio > 0.9, "saturation must be above the threshold")
finally:
stop.set()
thread.join(60)
log("restore default worker configuration")
t1 = time.time()
worker({})
resolved, _ = wait_alert(ALERT, "resolved", 240, t1)
drained = obs.wait_for(lambda: obs.prom_scalar('max(messenger_queue_depth{queue="attachments.process",state="queued"})',
default=-1) == 0, 240, 5)
log("backlog drained: %s (+%.0f s after restore)" % (bool(drained), time.time() - t1))
check(drained, "backlog must drain")
result.update({"alert_pending_s": pending, "alert_firing_s": round(firing), "saturation_at_firing": round(ratio, 3),
"queued_at_firing": depth, "oldest_age_s": round(age), "jobs_per_s": round(rate, 2),
"job_p95_s": round(p95, 2), "alerts_at_firing": others, "resolved_after_restore_s": round(resolved),
"drained_after_restore_s": round(time.time() - t1), "uploads": summary.get("status_counts")})
(ROOT / "evidence/worker_saturation.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: MessengerWorkerSaturated fired under saturation and resolved after recovery")
if __name__ == "__main__":
run(main)
+343
View File
@@ -0,0 +1,343 @@
#!/usr/bin/env python3
"""Track 5.1 "Цена наблюдаемости": the same open-loop load against several telemetry configurations.
Configurations (applied by recreating api-1, api-2, worker and otel-collector with shell env overrides):
off OTEL_SDK_DISABLED=true no SDK, no instrumentation (baseline)
on100 sampler ratio 1.0, collector profile default every span exported
head10 sampler ratio 0.1, collector profile default head sampling of root spans (children follow)
tail sampler ratio 1.0, collector profile tail balanced: errors + >500 ms + async pipeline + 10% rest
Per run: recreate -> ready -> warm-up 20 s -> idle 20 s -> counters A -> load (rate, duration) -> idle 30 s ->
counters B. Delta(B-A) = telemetry produced by the measured load. CPU is the exact cgroup delta over the load window
(Docker API); RSS is the maximum of `docker stats` samples taken every 5 s. Afterwards Tempo is searched in the load window for traces of injected slow/error requests and
upload pipelines, to count what survives sampling.
python3 student-tests/track1.py run [--configs off,on100,head10,tail] [--repeats 2] [--rate 60] [--duration 90]
python3 student-tests/track1.py summary # evidence/track1/results.jsonl -> evidence/track1/summary.md
"""
import argparse
import json
import os
import re
import statistics
import subprocess
import threading
import time
from pathlib import Path
import obs
from openloop import Load
from stand import Client, log
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence/track1"
CONFIGS = {
"off": {"OTEL_SDK_DISABLED": "true", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"},
"on100": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "default"},
"head10": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_PROFILE": "default"},
"tail": {"OTEL_SDK_DISABLED": "false", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_PROFILE": "tail"},
}
SERVICES = ("api-1", "api-2", "worker", "otel-collector", "tempo", "loki", "prometheus", "postgres", "nginx")
COUNTERS = {
# SDK side (sum over all instances; absent when the SDK is off)
"sdk_span_decisions_sample": 'sum(messenger_telemetry_sampler_decisions_total{decision="sample"})',
"sdk_span_decisions_drop_ratio": 'sum(messenger_telemetry_sampler_decisions_total{decision="drop",rule=~"ratio|parent"})',
"sdk_spans_exported_ok": 'sum(messenger_telemetry_spans_exported_total{outcome="success"})',
"sdk_spans_exported_failed": 'sum(messenger_telemetry_spans_exported_total{outcome="failure"})',
# Collector
"col_spans_in": "sum(otelcol_receiver_accepted_spans_total)",
"col_logs_in": "sum(otelcol_receiver_accepted_log_records_total)",
"col_points_in": "sum(otelcol_receiver_accepted_metric_points_total)",
"col_spans_out": 'sum(otelcol_exporter_sent_spans_total{exporter="otlp/tempo"})',
"col_spans_refused": "sum(otelcol_receiver_refused_spans_total)",
"col_spans_dropped": 'sum(otelcol_exporter_enqueue_failed_spans_total) or vector(0)',
"col_otlp_bytes_in": 'sum({__name__="http.server.request.size_bytes_total"})',
# Backends
"tempo_spans_in": "sum(tempo_distributor_spans_received_total)",
"tempo_bytes_in": "sum(tempo_distributor_bytes_received_total)",
"loki_bytes_in": "sum(loki_distributor_bytes_received_total)",
"loki_lines_in": "sum(loki_distributor_lines_received_total)",
"prom_samples_in": "sum(prometheus_tsdb_head_samples_appended_total)",
}
def compose(*args, env=None, timeout=300):
done = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, capture_output=True,
text=True, timeout=timeout, env={**os.environ, **(env or {})})
if done.returncode != 0:
raise RuntimeError("compose %s: %s" % (args[:2], done.stderr[-400:]))
return done.stdout
def apply(name):
env = CONFIGS[name]
compose("up", "-d", "--wait", "--force-recreate", "--no-deps", "otel-collector", "api-1", "api-2", "worker",
env=env)
compose("restart", "nginx") # re-resolve recreated replicas
c = Client()
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
break
except Exception:
time.sleep(1)
out = compose("exec", "-T", "api-1", "printenv", "OTEL_SDK_DISABLED", "OTEL_TRACES_SAMPLER_ARG")
log("config %s applied: api-1 env %s" % (name, out.split()))
def counters():
values = {}
for key, expr in COUNTERS.items():
try:
rows = obs.prom(expr)
values[key] = rows[0][1] if rows else None
except Exception:
values[key] = None
tail = {}
for metric, value in obs.prom('{__name__=~"otelcol_processor_tail_sampling_count_traces_sampled.*"}'):
tail["%s/%s" % (metric.get("policy"), metric.get("sampled"))] = value
values["tail_policies"] = tail
return values
def delta(a, b):
out = {}
for key in COUNTERS:
if a.get(key) is not None and b.get(key) is not None:
out[key] = b[key] - a[key]
elif b.get(key) is not None:
out[key] = b[key] # series appeared during the window (counter started at 0)
else:
out[key] = None
out["tail_policies"] = {k: v - a["tail_policies"].get(k, 0) for k, v in b["tail_policies"].items()}
return out
def parse_mem(text):
"""'123.4MiB' -> 123.4 (MiB)."""
match = re.match(r"\s*([\d.]+)\s*([A-Za-z]*)", text)
if not match:
return 0.0
scale = {"B": 1 / 1048576, "kB": 1 / 1024, "KiB": 1 / 1024, "MiB": 1, "MB": 1, "GiB": 1024, "GB": 1024}
return float(match.group(1)) * scale.get(match.group(2), 1)
def sample_stats(stop, samples):
while not stop.is_set():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
capture_output=True, text=True).stdout
row = {}
for line in out.splitlines():
name, cpu, mem = line.split("\t")
if not name.startswith("mtusi-lab6-"):
continue
svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0]
cur = row.setdefault(svc, [0.0, 0.0])
cur[0] += float(cpu.rstrip("%") or 0)
cur[1] += parse_mem(mem.split(" / ")[0])
samples.append(row)
stop.wait(5)
def docker_socket():
host = subprocess.run(["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"],
capture_output=True, text=True).stdout.strip()
return host.replace("unix://", "")
def cpu_seconds():
"""Cumulative container CPU seconds per service from the Docker API (exact over a window, unlike docker stats)."""
sock = docker_socket()
ids = subprocess.run(["docker", "ps", "--filter", "label=com.docker.compose.project=mtusi-lab6", "--format",
"{{.ID}} {{.Names}}"], capture_output=True, text=True).stdout.splitlines()
out = {}
for line in ids:
cid, name = line.split()
svc = name[len("mtusi-lab6-"):].rsplit("-", 1)[0]
raw = subprocess.run(["curl", "-s", "--unix-socket", sock,
"http://localhost/containers/%s/stats?stream=false&one-shot=true" % cid],
capture_output=True, text=True).stdout
out[svc] = out.get(svc, 0.0) + json.loads(raw)["cpu_stats"]["cpu_usage"]["total_usage"] / 1e9
return out
def count_traces(query, start, end):
try:
return len(obs.tempo_search(query, limit=10000, start=start, end=end))
except Exception as exc:
return "error:%s" % type(exc).__name__
def one_run(name, rep, rate, duration):
apply(name)
load = Load("http://localhost:8080", users=4, threads=96)
load.run(rate, 20, 0.5, 1, 1) # warm-up, same mix
time.sleep(20)
a = counters()
load = Load("http://localhost:8080", users=4, threads=96)
stop, samples = threading.Event(), []
sampler = threading.Thread(target=sample_stats, args=(stop, samples), daemon=True)
sampler.start()
cpu_a = cpu_seconds()
t_start = time.time()
summary = load.run(rate, duration, 1, 1, 1)
t_end = time.time()
cpu_b = cpu_seconds()
stop.set()
sampler.join(15)
time.sleep(30) # SDK batch (2 s) + collector batch (2 s) + tail decision_wait (10 s) + scrape (15 s)
b = counters()
d = delta(a, b)
# % of one core averaged over the load window (cumulative cgroup CPU; several replicas of a service are summed)
cpu = {svc: round((cpu_b[svc] - cpu_a[svc]) / (t_end - t_start) * 100, 2) for svc in cpu_b if svc in cpu_a}
mem = {svc: round(max(s[svc][1] for s in samples if svc in s), 1) for svc in samples[0]} if samples else {}
w0, w1 = int(t_start) - 2, int(t_end) + 15
traces = {
"server_api": count_traces('{resource.service.name="messenger-api" && kind=server}', w0, w1),
"slow": count_traces('{span.messenger.fault="slow"}', w0, w1),
"error": count_traces('{span.messenger.fault="error"}', w0, w1),
"pipeline_consumer": count_traces('{kind=consumer && span.messenger.job.outcome="succeeded"}', w0, w1),
}
result = {"config": name, "rep": rep, "env": CONFIGS[name], "load": summary, "cpu_pct_avg": cpu,
"mem_mib_max": mem, "telemetry_delta": d, "traces_in_tempo": traces,
"window": [t_start, t_end], "stats_samples": len(samples)}
log("%s#%d: ok_rps=%s p50/p95/p99=%s/%s/%s ms api_cpu=%s+%s%% spans_in=%s tempo_spans=%s slow=%s/%d err=%s/%d" % (
name, rep, summary["successful_rps"], summary["latency_ms_plain_ok"]["p50"],
summary["latency_ms_plain_ok"]["p95"], summary["latency_ms_plain_ok"]["p99"], cpu.get("api-1"),
cpu.get("api-2"), d.get("col_spans_in"), d.get("tempo_spans_in"), traces["slow"],
summary["faults_sent"]["slow"], traces["error"], summary["faults_sent"]["error"]))
return result
def cmd_run(args):
OUT.mkdir(parents=True, exist_ok=True)
path = OUT / "results.jsonl"
names = args.configs.split(",")
try:
for rep in range(1, args.repeats + 1):
for name in names:
result = one_run(name, rep, args.rate, args.duration)
result["hardware"] = HARDWARE
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(result, ensure_ascii=False) + "\n")
finally:
log("restore default configuration (on100)")
apply("on100")
def fmt(v, digits=0):
if v is None:
return "—"
if isinstance(v, str):
return v
return ("%%.%df" % digits) % v
def metric_500s(run_result):
"""Injected 500s as seen by METRICS for this run: metrics are computed in the SDK, before any trace sampling."""
start, end = run_result["window"]
if run_result["config"] == "off":
return None
# Exact counter difference at the window borders (increase() extrapolates and overstates by ~10% here).
expr = 'sum(http_server_request_duration_seconds_count{service_name="messenger-api",http_response_status_code="500"})'
try:
return obs.prom_scalar(expr, at=end + 20) - obs.prom_scalar(expr, at=start - 5)
except Exception:
return None
def cmd_summary(_args):
rows = [json.loads(line) for line in (OUT / "results.jsonl").read_text(encoding="utf-8").splitlines() if line]
by = {}
for r in rows:
by.setdefault(r["config"], []).append(r)
def values(items, fn):
return [v for v in (fn(x) for x in items) if isinstance(v, (int, float))]
def avg(items, fn):
vals = values(items, fn)
return statistics.mean(vals) if vals else None
def spread(items, fn, digits=1):
vals = values(items, fn)
if not vals:
return "—"
return "%s (%s…%s)" % (fmt(statistics.mean(vals), digits), fmt(min(vals), digits), fmt(max(vals), digits))
lat = lambda q: (lambda x: x["load"]["latency_ms_plain_ok"][q]) # noqa: E731
cpu = lambda *svc: (lambda x: sum(x["cpu_pct_avg"].get(s, 0) for s in svc)) # noqa: E731
delta_of = lambda key: (lambda x: x["telemetry_delta"].get(key)) # noqa: E731
traces = lambda key: (lambda x: x["traces_in_tempo"].get(key)) # noqa: E731
faults = lambda key: (lambda x: x["load"]["faults_sent"][key]) # noqa: E731
names = [n for n in CONFIGS if n in by]
lines = ["Среднее по прогонам (мин…макс). CPU — % одного ядра за окно нагрузки, реплики сервиса суммируются.", "",
"| Конфигурация | Прогонов | Успешн. RPS | Неожид. ошибки | p50, ms | p95, ms | p99, ms | CPU api-1+2, % | CPU worker, % |",
"|---|---|---|---|---|---|---|---|---|"]
for n in names:
i = by[n]
lines.append("| %s | %d | %s | %s | %s | %s | %s | %s | %s |" % (
n, len(i), fmt(avg(i, lambda x: x["load"]["successful_rps"]), 1),
fmt(avg(i, lambda x: x["load"]["server_errors_unexpected"])), spread(i, lat("p50")), spread(i, lat("p95")),
spread(i, lat("p99")), spread(i, cpu("api-1", "api-2")), spread(i, cpu("worker"))))
lines += ["", "| Конфигурация | CPU collector, % | CPU Tempo, % | CPU Loki, % | CPU Prometheus, % | RSS api-1 (макс), MiB | RSS collector, MiB | RSS Tempo, MiB |",
"|---|---|---|---|---|---|---|---|"]
for n in names:
i = by[n]
mem = lambda s: (lambda x: x["mem_mib_max"].get(s)) # noqa: E731
lines.append("| %s | %s | %s | %s | %s | %s | %s | %s |" % (
n, spread(i, cpu("otel-collector"), 2), spread(i, cpu("tempo"), 2), spread(i, cpu("loki"), 2),
spread(i, cpu("prometheus"), 2), fmt(avg(i, mem("api-1"))), fmt(avg(i, mem("otel-collector"))),
fmt(avg(i, mem("tempo")))))
lines += ["", "| Конфигурация | Spans → collector | Spans → Tempo | Отказы/отбросы collector | OTLP ingest, KiB | Tempo ingest, KiB | Loki ingest, KiB |",
"|---|---|---|---|---|---|---|"]
for n in names:
i = by[n]
kib = lambda key: (avg(i, delta_of(key)) / 1024 if avg(i, delta_of(key)) is not None else None) # noqa: E731
lost = avg(i, lambda x: (x["telemetry_delta"].get("col_spans_refused") or 0) + (x["telemetry_delta"].get("col_spans_dropped") or 0))
lines.append("| %s | %s | %s | %s | %s | %s | %s |" % (
n, fmt(avg(i, delta_of("col_spans_in"))), fmt(avg(i, delta_of("tempo_spans_in"))), fmt(lost),
fmt(kib("col_otlp_bytes_in")), fmt(kib("tempo_bytes_in")), fmt(kib("loki_bytes_in"))))
lines += ["", "Сумма по всем прогонам конфигурации: сохранено трасс / отправлено. «500 в метриках» — разность счётчика HTTP в Prometheus на границах окна.", "",
"| Конфигурация | Серверных трасс API | slow (+600 ms) | error (500) | upload-конвейеров | 500 в метриках / отправлено |",
"|---|---|---|---|---|---|"]
for n in names:
i = by[n]
total = lambda fn: sum(values(i, fn)) # noqa: E731
m500 = [metric_500s(x) for x in i]
lines.append("| %s | %d / %d | %d / %d | %d / %d | %d / %d | %s / %d |" % (
n, total(traces("server_api")), total(lambda x: x["load"]["requests"]),
total(traces("slow")), total(faults("slow")), total(traces("error")), total(faults("error")),
total(traces("pipeline_consumer")), total(lambda x: x["load"]["by_kind"]["upload"]["n"]),
"—" if None in m500 else "%.0f" % sum(m500), total(faults("error"))))
text = "\n".join(lines) + "\n"
(OUT / "summary.md").write_text(text, encoding="utf-8")
print(text)
HARDWARE = {}
def main():
p = argparse.ArgumentParser()
sub = p.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("run")
r.add_argument("--configs", default="off,on100,head10,tail")
r.add_argument("--repeats", type=int, default=2)
r.add_argument("--rate", type=float, default=60)
r.add_argument("--duration", type=float, default=90)
sub.add_parser("summary")
args = p.parse_args()
if args.cmd == "run":
info = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} {{.MemTotal}}"],
capture_output=True, text=True).stdout.split()
HARDWARE.update({"docker": info[0], "vm_cpus": int(info[1]), "vm_mem_gib": round(int(info[2]) / 2**30, 1)})
cmd_run(args)
else:
cmd_summary(args)
if __name__ == "__main__":
main()
+126
View File
@@ -0,0 +1,126 @@
"""Variant rules and worker resource limits. Runs INSIDE the worker container (Pillow + app code):
docker compose exec -T --index 1 worker python - < student-tests/variants_check.py
"""
import io
import json
import os
import resource
import subprocess
import sys
import tempfile
import time
from PIL import Image
sys.path.insert(0, "/app")
from app import config # noqa: E402
FAILS = []
def check(cond, msg):
print(("ok " if cond else "FAIL ") + msg, flush=True)
if not cond:
FAILS.append(msg)
def run_child(path, mem_mb=config.WORKER_CHILD_MEMORY_MB, cpu=config.JOB_TIMEOUT_SECONDS, max_pixels=config.MAX_IMAGE_PIXELS):
out = tempfile.mkdtemp()
def limits():
resource.setrlimit(resource.RLIMIT_AS, (mem_mb << 20, mem_mb << 20))
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
start = time.monotonic()
done = subprocess.run([sys.executable, "-m", "app.imageproc", path, out, str(max_pixels)], capture_output=True,
preexec_fn=limits, timeout=120, env={"PATH": os.environ["PATH"], "MALLOC_ARENA_MAX": "2"})
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
produced = {v["name"]: v for v in json.loads(done.stdout)} if done.returncode == 0 else {}
return done.returncode, produced, time.monotonic() - start, usage.ru_maxrss / 1024
def save(img, fmt, **kw):
path = tempfile.mktemp(suffix="." + fmt.lower())
img.save(path, fmt, **kw)
return path
def dims(v):
return v["width"], v["height"]
def real(v):
with Image.open(v["file"]) as im:
return im.format, im.size, "exif" in im.info, im
# 1. fixture 320x200 -> 256x160, crop 128, optimized not upscaled
rc, p, _, _ = run_child("/app/tests/fixtures/sample.png" if os.path.exists("/app/tests/fixtures/sample.png") else
save(Image.new("RGB", (320, 200), "red"), "PNG"))
check(rc == 0 and dims(p["thumbnail"]) == (256, 160), "320x200 -> thumbnail 256x160")
check(dims(p["crop"]) == (128, 128), "crop 128x128")
check(dims(p["optimized"]) == (320, 200) and real(p["optimized"])[0] == "WEBP", "optimized WebP 320x200, no upscale")
check(real(p["thumbnail"])[0] == "JPEG" and real(p["crop"])[0] == "JPEG", "thumbnail/crop are JPEG")
# 2. small source: no upscale for thumbnail, upscale allowed for crop
rc, p, _, _ = run_child(save(Image.new("RGB", (40, 10), "blue"), "PNG"))
check(dims(p["thumbnail"]) == (40, 10) and dims(p["crop"]) == (128, 128), "40x10: thumbnail 40x10, crop 128x128")
# 3. extreme aspect ratio: floor rounding, minimum 1 px
rc, p, _, _ = run_child(save(Image.new("RGB", (3000, 2), "blue"), "PNG"))
check(dims(p["thumbnail"]) == (256, 1) and dims(p["optimized"]) == (1280, 1), "3000x2 -> 256x1 / 1280x1 (min 1px)")
rc, p, _, _ = run_child(save(Image.new("RGB", (1001, 999), "blue"), "PNG"))
check(dims(p["thumbnail"]) == (256, 255), "1001x999 -> 256x255 (floor)")
# 4. EXIF orientation 6 (rotate 90): stored 400x200 -> upright 200x400 -> thumbnail 128x256; EXIF stripped
exif = Image.Exif()
exif[0x0112] = 6
img = Image.new("RGB", (400, 200), "white")
img.paste((255, 0, 0), (0, 0, 200, 200)) # left half red
rc, p, _, _ = run_child(save(img, "JPEG", exif=exif.tobytes(), quality=95))
fmt, size, has_exif, im = real(p["thumbnail"])
check(size == (128, 256), "EXIF orientation applied: 400x200 (o=6) -> 128x256")
with Image.open(p["thumbnail"]["file"]) as t:
top = t.convert("RGB").getpixel((64, 20))
check(top[0] > 200 and top[1] < 80, "rotated content: red half on top after orientation 6")
check(not has_exif and "exif" not in Image.open(p["optimized"]["file"]).info, "EXIF removed from thumbnail/optimized")
# 5. alpha composited on white for JPEG, kept for WebP
rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0))
rc, p, _, _ = run_child(save(rgba, "PNG"))
with Image.open(p["thumbnail"]["file"]) as t:
px = t.convert("RGB").getpixel((50, 50))
check(min(px) >= 250, "transparent PNG -> white JPEG thumbnail")
check(Image.open(p["optimized"]["file"]).mode == "RGBA", "WebP optimized keeps alpha")
# 6. 16-bit grayscale PNG, CMYK JPEG, palette PNG
rc16, _, _, _ = run_child(save(Image.new("I;16", (300, 300), 40000), "PNG"))
rcc, _, _, _ = run_child(save(Image.new("CMYK", (300, 300), (0, 255, 0, 0)), "JPEG"))
rcp, _, _, _ = run_child(save(Image.new("P", (300, 300), 3), "PNG", transparency=3))
check(rc16 == 0 and rcc == 0 and rcp == 0, "16-bit PNG, CMYK JPEG, palette+transparency PNG are processed")
# 7. limits: pixels (exit 4), not an image (exit 3), memory (exit 5), CPU (SIGXCPU)
rc, _, _, _ = run_child(save(Image.new("RGB", (100, 100)), "PNG"), max_pixels=5000)
check(rc == 4, "pixel limit enforced before decode -> image_too_large (exit 4)")
junk = tempfile.mktemp()
open(junk, "wb").write(b"\x89PNG\r\n\x1a\n" + os.urandom(2000))
rc, _, _, _ = run_child(junk)
check(rc == 3, "corrupted PNG -> invalid_image (exit 3)")
big = save(Image.new("RGB", (6000, 6000), "green"), "PNG")
rc, _, _, _ = run_child(big, mem_mb=160)
check(rc == 5, "36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got %s" % rc)
# 8. largest allowed inputs under the real limits: time and child peak RSS
for name, w, h, fmt in (("png", 7900, 5000, "PNG"), ("jpeg", 7900, 5000, "JPEG")):
noise = Image.effect_noise((w // 10, h // 10), 60).convert("RGB").resize((w, h))
path = save(noise, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 1}))
rc, p, secs, rss = run_child(path)
check(rc == 0 and dims(p["thumbnail"]) == (256, 162), "%.1f Mpx %s (%d MB file) ok in %.1fs, peak child RSS %.0f MiB"
% (w * h / 1e6, name.upper(), os.path.getsize(path) >> 20, secs, rss))
if fmt == "PNG":
rc, _, _, _ = run_child(path, cpu=0)
check(rc == -24, "CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got %s" % rc)
print("RESULT:", "PASS" if not FAILS else "FAIL %s" % FAILS)
sys.exit(1 if FAILS else 0)